Merge branch 'main' into sam/netp-debug-command-disable-vpn

* main: add alternateHtmlLoad navigation type (HTML Error Page) (#644) Check subfeature state before reading rollout data (#649) Add NavigationPreferences CustomHeaderFields (#651) Update autofill to 10.1.0 (#643) Add Autoconsent onByDefault subfeature (#647) Add error codes to site breakage reports (#642) Run unit tests also on iOS Simulator (#641) Improve VPN auth token storage (#639) Bump content-scope-scripts to 4.59.2 (#638) Fix `site:` queries escaping with iOS 17 SDK (#640) Breakage improvement (#621) Don't report CancellationError from BookmarksFaviconsFetcher (#634) Add explicit mapping of SyncError to error code (#637)
duckduckgo · Feb 8, 2024 · 7a3af2c · 7a3af2c
2 parents 14329a0 + 51b5d28
commit 7a3af2c
Show file tree

Hide file tree

Showing 67 changed files with 1,651 additions and 256 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -20,7 +20,7 @@ jobs:
 
   unit-tests:
 
-      name: Run unit tests
+      name: Run unit tests (macOS)
 
       runs-on: macos-13-xlarge
       timeout-minutes: 30
@@ -71,7 +71,7 @@ jobs:
         uses: mikepenz/action-junit-report@v3
         if: always()
         with:
-          check_name: Test Report
+          check_name: Test Report (macOS)
           report_paths: tests.xml
           require_tests: true
 
@@ -105,9 +105,96 @@ jobs:
           author=$(gh api https://api.github.com/repos/${{ github.repository }}/commits/${head_commit} --jq .author.login)
           echo "commit_author=${author}" >> $GITHUB_OUTPUT
 
+  unit-tests-ios:
+
+      name: Run unit tests (iOS)
+
+      runs-on: macos-13-xlarge
+      timeout-minutes: 30
+
+      steps:
+
+      - name: Check out the code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Set cache key hash
+        run: |
+          has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' Package.resolved)
+          if [[ "$has_only_tags" == "true" ]]; then
+            echo "cache_key_hash=${{ hashFiles('Package.resolved') }}" >> $GITHUB_ENV
+          else
+            echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+          fi
+
+      - name: Cache SPM
+        if: env.cache_key_hash
+        uses: actions/cache@v3
+        with:
+          path: DerivedData/SourcePackages
+          key: ${{ runner.os }}-spm-ios-${{ env.cache_key_hash }}
+          restore-keys: |
+            ${{ runner.os }}-spm-ios-
+
+      - name: Select Xcode
+        run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
+
+      - name: Resolve package dependencies
+        run: |
+          while xcodebuild -resolvePackageDependencies \
+            -scheme BrowserServicesKit-Package \
+            -destination 'platform=iOS Simulator,name=iPhone 15' \
+            -derivedDataPath DerivedData \
+            2>&1 | grep Error; do :; done
+
+          # you just can't have good things
+          # set -o pipefail && swift package resolve | tee ios-build-log.txt | xcbeautify
+          # mkdir -p DerivedData/SourcePackages
+          # mv .build/* DerivedData/SourcePackages
+
+      - name: Run tests
+        run: |
+          set -o pipefail && xcodebuild test \
+            -scheme BrowserServicesKit \
+            -destination 'platform=iOS Simulator,name=iPhone 15' \
+            -derivedDataPath DerivedData \
+            -skipPackagePluginValidation \
+            CODE_SIGNING_ALLOWED=NO \
+            | tee -a ios-build-log.txt \
+            | xcbeautify --report junit --report-path . --junit-report-filename ios-unittests.xml 
+
+      - name: Publish Unit Tests Report
+        uses: mikepenz/action-junit-report@v3
+        if: always()
+        with:
+          check_name: Test Report (iOS)
+          report_paths: ios-unittests.xml
+          require_tests: true
+
+      - name: Update Asana with failed unit tests
+        if: always() # always run even if the previous step fails
+        env:
+          ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
+          WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
+        run: |
+          # Extract failed tests from the junit report
+          # Only keep failures unique by classname and name (column 1 and 2 of the yq output)
+          yq < ios-unittests.xml -p xml -o json -r \
+            $'[.testsuites.testsuite[].testcase] | flatten | map(select(.failure) | .+@classname + " " + .+@name + " \'" + .failure.+@message + "\' ${{ env.WORKFLOW_URL }}") | .[]' \
+            | sort -u -k 1,2 \
+            | xargs -L 1 ./scripts/report-failed-unit-test.sh -s ${{ vars.APPLE_CI_FAILING_TESTS_FAILED_TESTS_SECTION_ID }}
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ios-build-log.txt
+          path: ios-build-log.txt
+
   create-asana-task:
     name: Create Asana Task
-    needs: [swiftlint, unit-tests]
+    needs: [swiftlint, unit-tests, unit-tests-ios]
 
     if: failure() && github.ref_name == 'main' && github.run_attempt == 1
 
@@ -126,7 +213,7 @@ jobs:
 
   close-asana-task:
     name: Close Asana Task
-    needs: [swiftlint, unit-tests]
+    needs: [swiftlint, unit-tests, unit-tests-ios]
 
     if: success() && github.ref_name == 'main' && github.run_attempt > 1
 

diff --git a/Package.resolved b/Package.resolved
@@ -14,17 +14,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "bb027f14bec7fbb1a85d308139e7a66686da160e",
-        "version" : "4.59.0"
+        "revision" : "063b560e59a50e03d9b00b88a7fcb2ed2b562395",
+        "version" : "4.61.0"
       }
     },
     {
       "identity" : "duckduckgo-autofill",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/duckduckgo-autofill.git",
       "state" : {
-        "revision" : "b972bc0ab6ee1d57a0a18a197dcc31e40ae6ac57",
-        "version" : "10.0.3"
+        "revision" : "03d3e3a959dd75afbe8c59b5a203ea676d37555d",
+        "version" : "10.1.0"
       }
     },
     {
@@ -41,8 +41,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/privacy-dashboard",
       "state" : {
-        "revision" : "38336a574e13090764ba09a6b877d15ee514e371",
-        "version" : "3.1.1"
+        "revision" : "c67d268bf234760f49034a0fe7a6137a1b216b05",
+        "version" : "3.2.0"
       }
     },
     {

diff --git a/Package.swift b/Package.swift
@@ -34,13 +34,13 @@ let package = Package(
         .plugin(name: "SwiftLintPlugin", targets: ["SwiftLintPlugin"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "10.0.3"),
+        .package(url: "https://github.com/duckduckgo/duckduckgo-autofill.git", exact: "10.1.0"),
         .package(url: "https://github.com/duckduckgo/GRDB.swift.git", exact: "2.3.0"),
         .package(url: "https://github.com/duckduckgo/TrackerRadarKit", exact: "1.2.2"),
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "2.1.0"),
-        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "3.1.1" ),
-        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "4.59.1"),
+        .package(url: "https://github.com/duckduckgo/privacy-dashboard", exact: "3.2.0"),
+        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "4.61.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/wireguard-apple", exact: "1.1.1"),
@@ -178,6 +178,7 @@ let package = Package(
                 .define("_FRAME_HANDLE_ENABLED", .when(platforms: [.macOS])),
                 .define("PRIVATE_NAVIGATION_DID_FINISH_CALLBACKS_ENABLED", .when(platforms: [.macOS])),
                 .define("TERMINATE_WITH_REASON_ENABLED", .when(platforms: [.macOS])),
+                .define("_WEBPAGE_PREFS_CUSTOM_HEADERS_ENABLED", .when(platforms: [.macOS])),
             ],
             plugins: [.plugin(name: "SwiftLintPlugin")]
         ),
@@ -198,6 +199,7 @@ let package = Package(
                 "TrackerRadarKit",
                 "UserScript",
                 "ContentBlocking",
+                "Persistence",
                 .product(name: "PrivacyDashboardResources", package: "privacy-dashboard")
             ],
             path: "Sources/PrivacyDashboard",
@@ -381,6 +383,7 @@ let package = Package(
                 .define("_IS_USER_INITIATED_ENABLED", .when(platforms: [.macOS])),
                 .define("_FRAME_HANDLE_ENABLED", .when(platforms: [.macOS])),
                 .define("PRIVATE_NAVIGATION_DID_FINISH_CALLBACKS_ENABLED", .when(platforms: [.macOS])),
+                .define("_WEBPAGE_PREFS_CUSTOM_HEADERS_ENABLED", .when(platforms: [.macOS])),
             ],
             plugins: [.plugin(name: "SwiftLintPlugin")]
         ),
@@ -452,6 +455,14 @@ let package = Package(
             ],
             plugins: [.plugin(name: "SwiftLintPlugin")]
         ),
+        .testTarget(
+            name: "PrivacyDashboardTests",
+            dependencies: [
+                "PrivacyDashboard",
+                "TestUtils"
+            ],
+            plugins: [.plugin(name: "SwiftLintPlugin")]
+        ),
     ],
     cxxLanguageStandard: .cxx11
 )

diff --git a/Sources/Bookmarks/FaviconsFetcher/BookmarksFaviconsFetcher.swift b/Sources/Bookmarks/FaviconsFetcher/BookmarksFaviconsFetcher.swift
@@ -199,7 +199,7 @@ public final class BookmarksFaviconsFetcher {
                 self?.fetchingDidFinishSubject.send(.failure(error))
                 if let fetcherError = error as? BookmarksFaviconsFetcherError {
                     self?.errorEvents?.fire(fetcherError)
-                } else {
+                } else if !(error is CancellationError) {
                     self?.errorEvents?.fire(.other(error))
                 }
             } else {

diff --git a/Sources/BookmarksTestDBBuilder/BookmarksTestDBBuilder.swift b/Sources/BookmarksTestDBBuilder/BookmarksTestDBBuilder.swift
@@ -16,7 +16,6 @@
 //  limitations under the License.
 //
 
-import Cocoa
 import Foundation
 import CoreData
 import Persistence

diff --git a/Sources/BrowserServicesKit/ContentScopeScript/SpecialPagesUserScript.swift b/Sources/BrowserServicesKit/ContentScopeScript/SpecialPagesUserScript.swift
@@ -29,7 +29,7 @@ public final class SpecialPagesUserScript: NSObject, UserScript, UserScriptMessa
     public static let context = "specialPages"
 
     // special pages messaging cannot be isolated as we'll want regular page-scripts to be able to communicate
-    public let broker = UserScriptMessageBroker(context: SpecialPagesUserScript.context, requiresRunInPageContentWorld: true )
+    public let broker = UserScriptMessageBroker(context: SpecialPagesUserScript.context, requiresRunInPageContentWorld: true)
 
     public let messageNames: [String] = [
         SpecialPagesUserScript.context

diff --git a/Sources/BrowserServicesKit/PrivacyConfig/AppPrivacyConfiguration.swift b/Sources/BrowserServicesKit/PrivacyConfig/AppPrivacyConfiguration.swift
@@ -195,27 +195,24 @@ public struct AppPrivacyConfiguration: PrivacyConfiguration {
         let subfeatures = subfeatures(for: subfeature.parent)
         let subfeatureData = subfeatures[subfeature.rawValue]
 
-        // Handle Rollouts
-        if let rollout = subfeatureData?.rollout {
-            if !isRolloutEnabled(subfeature: subfeature, rolloutSteps: rollout.steps, randomizer: randomizer) {
-                return .disabled(.stillInRollout)
-            }
-        }
-
         let satisfiesMinVersion = satisfiesMinVersion(subfeatureData?.minSupportedVersion, versionProvider: versionProvider)
 
         switch subfeatureData?.state {
         case PrivacyConfigurationData.State.enabled:
             guard satisfiesMinVersion else { return .disabled(.appVersionNotSupported) }
-
-            return .enabled
         case PrivacyConfigurationData.State.internal:
             guard internalUserDecider.isInternalUser else { return .disabled(.limitedToInternalUsers) }
             guard satisfiesMinVersion else { return .disabled(.appVersionNotSupported) }
-
-            return .enabled
         default: return .disabled(.disabledInConfig)
         }
+
+        // Handle Rollouts
+        if let rollout = subfeatureData?.rollout,
+           !isRolloutEnabled(subfeature: subfeature, rolloutSteps: rollout.steps, randomizer: randomizer) {
+            return .disabled(.stillInRollout)
+        }
+
+        return .enabled
     }
 
     private func subfeatures(for feature: PrivacyFeature) -> PrivacyConfigurationData.PrivacyFeature.Features {

diff --git a/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift b/Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift
@@ -94,3 +94,11 @@ public enum SyncSubfeature: String, PrivacySubfeature {
     case level2AllowSetupFlows
     case level3AllowCreateAccount
 }
+
+public enum AutoconsentSubfeature: String, PrivacySubfeature {
+    public var parent: PrivacyFeature {
+        .autoconsent
+    }
+
+    case onByDefault
+}
diff --git a/...es/BrowserServicesKit/SmarterEncryption/Store/HTTPSUpgrade.xcdatamodeld/.xccurrentversion b/...es/BrowserServicesKit/SmarterEncryption/Store/HTTPSUpgrade.xcdatamodeld/.xccurrentversion
@@ -1,5 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
-<dict/>
+<dict>
+	<key>_XCCurrentVersionName</key>
+	<string>HTTPSUpgrade 3.xcdatamodel</string>
+</dict>
 </plist>
diff --git a/Sources/Common/Extensions/URLExtension.swift b/Sources/Common/Extensions/URLExtension.swift
@@ -158,21 +158,16 @@ extension URL {
             s = scheme.separated() + s.dropFirst(scheme.separated().count - 1)
         }
 
-#if os(macOS)
         let url: URL?
         let urlWithScheme: URL?
-        if #available(macOS 14.0, *) {
+        if #available(macOS 14.0, iOS 17.0, *) {
             // Making sure string is strictly valid according to the RFC
             url = URL(string: s, encodingInvalidCharacters: false)
             urlWithScheme = URL(string: NavigationalScheme.http.separated() + s, encodingInvalidCharacters: false)
         } else {
             url = URL(string: s)
             urlWithScheme = URL(string: NavigationalScheme.http.separated() + s)
         }
-#else
-        let url = URL(string: s)
-        let urlWithScheme = URL(string: NavigationalScheme.http.separated() + s)
-#endif
 
         if let url {
             // if URL has domain:port or user:password@domain mistakengly interpreted as a scheme

diff --git a/Sources/DDGSync/DDGSync.swift b/Sources/DDGSync/DDGSync.swift
@@ -336,7 +336,7 @@ public class DDGSync: DDGSyncing {
         } catch {
             // swiftlint:disable:next line_length
             os_log(.error, log: dependencies.log, "Failed to delete account upon unauthenticated server response: %{public}s", error.localizedDescription)
-            if let syncError = error as? SyncError {
+            if error is SyncError {
                 throw error
             }
         }

diff --git a/Sources/DDGSync/SyncError.swift b/Sources/DDGSync/SyncError.swift
@@ -140,3 +140,50 @@ public enum SyncError: Error, Equatable {
         }
     }
 }
+
+extension SyncError: CustomNSError {
+
+    /**
+     * The mapping here can't be changed, or it will skew usage metrics.
+     */
+    public var errorCode: Int {
+        switch self {
+        case .noToken: return 13
+
+        case .failedToMigrate: return 14
+        case .failedToLoadAccount: return 15
+        case .failedToSetupEngine: return 16
+
+        case .failedToCreateAccountKeys: return 0
+        case .accountNotFound: return 17
+        case .accountAlreadyExists: return 18
+        case .invalidRecoveryKey: return 19
+
+        case .noFeaturesSpecified: return 20
+        case .noResponseBody: return 21
+        case .unexpectedStatusCode: return 1
+        case .unexpectedResponseBody: return 22
+        case .unableToEncodeRequestBody: return 2
+        case .unableToDecodeResponse: return 3
+        case .invalidDataInResponse: return 4
+        case .accountRemoved: return 23
+
+        case .failedToEncryptValue: return 5
+        case .failedToDecryptValue: return 6
+        case .failedToPrepareForConnect: return 7
+        case .failedToOpenSealedBox: return 8
+        case .failedToSealData: return 9
+
+        case .failedToWriteSecureStore: return 10
+        case .failedToReadSecureStore: return 11
+        case .failedToRemoveSecureStore: return 12
+
+        case .credentialsMetadataMissingBeforeFirstSync: return 24
+        case .receivedCredentialsWithoutUUID: return 25
+        case .emailProtectionUsernamePresentButTokenMissing: return 26
+        case .settingsMetadataNotPresent: return 27
+        case .unauthenticatedWhileLoggedIn: return 28
+        }
+    }
+
+}
diff --git a/Sources/Navigation/DistributedNavigationDelegate.swift b/Sources/Navigation/DistributedNavigationDelegate.swift
@@ -519,15 +519,14 @@ extension DistributedNavigationDelegate: WKNavigationDelegate {
             // regular flow: start .expected navigation
             navigation = expectedNavigation
         } else if webView.url?.isEmpty == false {
-            assert(webView.url?.navigationalScheme == .about, "session restoration happening without NavigationAction")
             navigation = Navigation(identity: NavigationIdentity(wkNavigation), responders: responders, state: .expected(nil), isCurrent: true)
             if let finishedNavigation = wkNavigation?.navigation {
                 assert(finishedNavigation.state == .finished)
                 // about: scheme navigation for new window sometimes duplicates didStart/didCommit/didFinish events with the same WKNavigation
                 let navigationAction = NavigationAction(request: finishedNavigation.request, navigationType: finishedNavigation.navigationAction.navigationType, currentHistoryItemIdentity: nil, redirectHistory: nil, isUserInitiated: false, sourceFrame: finishedNavigation.navigationAction.sourceFrame, targetFrame: finishedNavigation.navigationAction.targetFrame, shouldDownload: false, mainFrameNavigation: navigation)
                 navigation.navigationActionReceived(navigationAction)
             } else {
-                navigation.navigationActionReceived(.sessionRestoreNavigation(webView: webView, mainFrameNavigation: navigation))
+                navigation.navigationActionReceived(.alternateHtmlLoadNavigation(webView: webView, mainFrameNavigation: navigation))
             }
             navigation.willStart()
         } else if let wkNavigation {