[navigation #2] AdClick attribution navigation Tab Extension (#886)

* AdClickAttributionTabExtension+NavigationResponder

* fix TabExtension test overrides, adClick extension tests

* fix AdClickAttributionTabExtensionTests teardown

* cleanup

* don‘t use shared state in tests

* fix adClick inherited attribution initialization

* cleanup

* Adjust adClick tests to better match UserScripts/UserContentController behaviour; cleanup

* fix RELEASE

* remove Swifter dependency

* fix file header

* [navigation #3+4] canGoBack/GoForward; SERP headers (#885)

* canGoBack, canGoForward RePublished, Tab.publishers refactored

* redo redirect(_:NavigationAction, invalidatingBackItemIfNeeded)

* tests

* don‘t use shared state in tests

* fix RELEASE

* rollback renaming

* drop RePublished

* fix header name

* [navigation #4] SERP headers navigation responder (#887)

* SERP headers handling in SerpHeadersNavigationResponder

* convert TabTests to custom SchemeHandler

Author: mallexxx

.swiftlint.tests.yml

@@ -14,6 +14,7 @@ disabled_rules:
   - cyclomatic_complexity
   - identifier_name
   - implicit_getter
+  - trailing_comma
 
 opt_in_rules:
   - file_header

DuckDuckGo.xcodeproj/project.pbxproj

@@ -16,13 +16,15 @@
 //  limitations under the License.
 //
 
-import os.log
+import BrowserServicesKit
 import Combine
 import Common
 import ContentBlocking
 import Foundation
-import BrowserServicesKit
+import Navigation
+import os.log
 import PrivacyDashboard
+import TrackerRadarKit
 import WebKit
 
 protocol AdClickAttributionDependencies {
@@ -46,18 +48,48 @@ protocol UserContentControllerProtocol: AnyObject {
     func installLocalContentRuleList(_ ruleList: WKContentRuleList, identifier: String)
 }
 
+protocol AdClickAttributionDetecting {
+    func onStartNavigation(url: URL?)
+    func on2XXResponse(url: URL?)
+    func onDidFinishNavigation(url: URL?)
+    func onDidFailNavigation()
+}
+extension AdClickAttributionDetection: AdClickAttributionDetecting {}
+
+protocol AdClickLogicProtocol: AnyObject {
+    var state: AdClickAttributionLogic.State { get }
+    var delegate: AdClickAttributionLogicDelegate? { get set }
+
+    func applyInheritedAttribution(state: AdClickAttributionLogic.State?)
+    func onRulesChanged(latestRules: [ContentBlockerRulesManager.Rules])
+    func onRequestDetected(request: DetectedRequest)
+
+    func onBackForwardNavigation(mainFrameURL: URL?)
+    func onProvisionalNavigation() async
+    func onDidFinishNavigation(host: String?, currentTime: Date)
+}
+extension AdClickAttributionLogic: AdClickLogicProtocol {}
+
+protocol ContentBlockerScriptProtocol: AnyObject {
+    var currentAdClickAttributionVendor: String? { get set }
+    var supplementaryTrackerData: [TrackerData] { get set }
+}
+extension ContentBlockerRulesUserScript: ContentBlockerScriptProtocol {}
+
 final class AdClickAttributionTabExtension: TabExtension {
 
-    private static func makeAdClickAttributionDetection(with dependencies: some AdClickAttributionDependencies) -> AdClickAttributionDetection {
-        return AdClickAttributionDetection(feature: dependencies.adClickAttribution,
-                                           tld: dependencies.tld,
-                                           eventReporting: dependencies.attributionEvents,
-                                           errorReporting: dependencies.attributionDebugEvents,
-                                           log: OSLog.attribution)
+    private static func makeAdClickAttributionDetection(with dependencies: any AdClickAttributionDependencies, delegate: AdClickAttributionLogic) -> AdClickAttributionDetection {
+        let detection = AdClickAttributionDetection(feature: dependencies.adClickAttribution,
+                                                    tld: dependencies.tld,
+                                                    eventReporting: dependencies.attributionEvents,
+                                                    errorReporting: dependencies.attributionDebugEvents,
+                                                    log: OSLog.attribution)
+        detection.delegate = delegate
+        return detection
 
     }
 
-    private static func makeAdClickAttributionLogic(with dependencies: some AdClickAttributionDependencies) -> AdClickAttributionLogic {
+    private static func makeAdClickAttributionLogic(with dependencies: any AdClickAttributionDependencies) -> AdClickAttributionLogic {
         return AdClickAttributionLogic(featureConfig: dependencies.adClickAttribution,
                                        rulesProvider: dependencies.adClickAttributionRulesProvider,
                                        tld: dependencies.tld,
@@ -66,33 +98,40 @@ final class AdClickAttributionTabExtension: TabExtension {
                                        log: OSLog.attribution)
     }
 
+    private static func makeAdClickAttribution(with dependencies: any AdClickAttributionDependencies) -> (AdClickLogicProtocol, AdClickAttributionDetecting) {
+        let logic = makeAdClickAttributionLogic(with: dependencies)
+        let detection = makeAdClickAttributionDetection(with: dependencies, delegate: logic)
+        return (logic, detection)
+    }
+
     private let dependencies: any AdClickAttributionDependencies
 
     private weak var userContentController: UserContentControllerProtocol?
-    private weak var contentBlockerRulesScript: ContentBlockerRulesUserScript?
+    private weak var contentBlockerRulesScript: ContentBlockerScriptProtocol?
+    private let dateTimeProvider: () -> Date
 
-    // to be made private
-    let detection: AdClickAttributionDetection
-    let logic: AdClickAttributionLogic
+    private let detection: AdClickAttributionDetecting
+    private let logic: AdClickLogicProtocol
 
-    public var currentAttributionState: AdClickAttributionLogic.State? {
+    public var currentAttributionState: AdClickAttributionLogic.State {
         logic.state
     }
 
     private var cancellables = Set<AnyCancellable>()
 
     init(inheritedAttribution: AdClickAttributionLogic.State?,
          userContentControllerFuture: Future<UserContentControllerProtocol, Never>,
-         contentBlockerRulesScriptPublisher: some Publisher<ContentBlockerRulesUserScript?, Never>,
+         contentBlockerRulesScriptPublisher: some Publisher<(any ContentBlockerScriptProtocol)?, Never>,
          trackerInfoPublisher: some Publisher<DetectedRequest, Never>,
-         dependencies: some AdClickAttributionDependencies) {
+         dependencies: some AdClickAttributionDependencies,
+         dateTimeProvider: @escaping () -> Date = Date.init,
+         logicsProvider: (AdClickAttributionDependencies) -> (AdClickLogicProtocol, AdClickAttributionDetecting) = AdClickAttributionTabExtension.makeAdClickAttribution) {
 
         self.dependencies = dependencies
-        self.detection = Self.makeAdClickAttributionDetection(with: dependencies)
-        self.logic = Self.makeAdClickAttributionLogic(with: dependencies)
+        self.dateTimeProvider = dateTimeProvider
 
-        logic.delegate = self
-        detection.delegate = logic
+        (self.logic, self.detection) = logicsProvider(dependencies)
+        self.logic.delegate = self
 
         // delay firing up until UserContentController is published
         userContentControllerFuture.sink { [weak self] userContentController in
@@ -103,7 +142,7 @@ final class AdClickAttributionTabExtension: TabExtension {
         }.store(in: &cancellables)
     }
 
-    private func delayedInitialization(with userContentController: UserContentControllerProtocol, inheritedAttribution: AdClickAttributionLogic.State?, contentBlockerRulesScriptPublisher: some Publisher<ContentBlockerRulesUserScript?, Never>, trackerInfoPublisher: some Publisher<DetectedRequest, Never>) {
+    private func delayedInitialization(with userContentController: UserContentControllerProtocol, inheritedAttribution: AdClickAttributionLogic.State?, contentBlockerRulesScriptPublisher: some Publisher<(any ContentBlockerScriptProtocol)?, Never>, trackerInfoPublisher: some Publisher<DetectedRequest, Never>) {
 
         cancellables.removeAll()
         self.userContentController = userContentController
@@ -173,14 +212,48 @@ extension AdClickAttributionTabExtension: AdClickAttributionLogicDelegate {
 
 }
 
-extension AppContentBlocking: AdClickAttributionDependencies {}
+extension AdClickAttributionTabExtension: NavigationResponder {
+
+    func decidePolicy(for navigationAction: NavigationAction, preferences: inout NavigationPreferences) async -> NavigationActionPolicy? {
+        if navigationAction.isForMainFrame, navigationAction.navigationType.isBackForward {
+            logic.onBackForwardNavigation(mainFrameURL: navigationAction.url)
+        }
+        return .next
+    }
+
+    func didStart(_ navigation: Navigation) {
+        detection.onStartNavigation(url: navigation.url)
+    }
+
+    func decidePolicy(for navigationResponse: NavigationResponse) async -> NavigationResponsePolicy? {
+        if navigationResponse.isForMainFrame,
+           let currentNavigation = navigationResponse.mainFrameNavigation,
+           navigationResponse.isSuccessful == true {
+            detection.on2XXResponse(url: currentNavigation.url)
+        }
+
+        await logic.onProvisionalNavigation()
+
+        return .next
+    }
+
+    func navigationDidFinish(_ navigation: Navigation) {
+        guard navigation.isCurrent else { return }
+        detection.onDidFinishNavigation(url: navigation.url)
+        logic.onDidFinishNavigation(host: navigation.url.host, currentTime: dateTimeProvider())
+    }
+
+    func navigation(_ navigation: Navigation, didFailWith error: WKError) {
+        guard navigation.isCurrent else { return }
+        detection.onDidFailNavigation()
+    }
+
+}
 
-protocol AdClickAttributionProtocol {
-    var currentAttributionState: AdClickAttributionLogic.State? { get }
+extension AppContentBlocking: AdClickAttributionDependencies {}
 
-    // to be removed
-    var detection: AdClickAttributionDetection { get }
-    var logic: AdClickAttributionLogic { get }
+protocol AdClickAttributionProtocol: AnyObject, NavigationResponder {
+    var currentAttributionState: AdClickAttributionLogic.State { get }
 }
 
 extension AdClickAttributionTabExtension: AdClickAttributionProtocol {

DuckDuckGo/Browser Tab/Extensions/AdClickAttributionTabExtension.swift

@@ -102,7 +102,7 @@ extension TabExtensionsBuilder {
         add {
             AdClickAttributionTabExtension(inheritedAttribution: args.inheritedAttribution,
                                            userContentControllerFuture: args.userContentControllerFuture,
-                                           contentBlockerRulesScriptPublisher: userScripts.map(\.?.contentBlockerRulesScript),
+                                           contentBlockerRulesScriptPublisher: userScripts.map { $0?.contentBlockerRulesScript },
                                            trackerInfoPublisher: trackerInfoPublisher,
                                            dependencies: dependencies.privacyFeatures.contentBlocking)
         }

DuckDuckGo/Browser Tab/Extensions/TabExtensions.swift

@@ -0,0 +1,51 @@
+//
+//  NavigationActionPolicyExtension.swift
+//
+//  Copyright © 2023 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Navigation
+
+extension NavigationActionPolicy {
+
+    /// cancel+redirect Navigation Action popping last WebView Back Item
+    /// if a client-redirected navigation has been committed its BackForwardItem will stay in history
+    /// when the Navigation Action is cancelled in decidePolicyForNavigationAction:
+    /// https://app.asana.com/0/inbox/1199237043628108/1201280322539473/1201353436736961
+    @MainActor
+    static func redirect(_ navigationAction: NavigationAction, invalidatingBackItemIfNeededFor webView: WebView, do redirect: @escaping (Navigator) -> Void) -> NavigationActionPolicy {
+        guard let mainFrame = navigationAction.mainFrameTarget else {
+            assertionFailure("Trying to redirect non-main-frame NavigationAction")
+            return .cancel
+        }
+        return .redirect(mainFrame) { navigator in
+            // Cancelled & Upgraded Client Redirect URL leaves wrong backForwardList record
+
+            if case .redirect(.client(delay: 0)) = navigationAction.navigationType,
+               // initial NavigationAction BackForwardListItem is not the Current Item (new item was pushed during navigation)
+               let fromHistoryItemIdentity = navigationAction.redirectHistory?.last?.fromHistoryItemIdentity,
+               fromHistoryItemIdentity != webView.backForwardList.currentItem?.identity {
+
+                navigator.goBack()?.overrideResponders { _, _ in
+                    // don‘t perform actual navigation, just pop the back item
+                    .cancel
+                }
+            }
+
+            redirect(navigator)
+        }
+    }
+
+}

DuckDuckGo/Browser Tab/Model/NavigationActionPolicyExtension.swift

@@ -32,7 +32,8 @@ extension Tab: NavigationResponder {
         navigationDelegate.setResponders(
             .weak(self),
 
-            // ...
+            .weak(nullable: self.adClickAttribution),
+            .struct(SerpHeadersNavigationResponder()),
 
             // should be the last, for Unit Tests navigation events tracking
             .struct(nullable: testsClosureNavigationResponder)