Merge branch 'main' into sam/remove-atb-from-default-pixel-parameters

* main: (41 commits)
  Update BSK with autofill 10.1.0 (#2414)
  Bump submodules/privacy-reference-tests from `a3acc21` to `6b7ad1e` (#2408)
  Add Autoconsent onByDefault subfeature (#2423)
  Metadata reverted https://app.asana.com/0/0/1206476969305202/1206481077004998/f
  metadata updated
  Release 7.107.0-0 (#2421)
  More script fixes (#2419)
  More release script fixes (#2418)
  Release/7.106.0-4 (#2417)
  Reenable toggle on disallowing vpn (#2404) (#2416)
  Don't crash when AppRatingPromptEntity fetch errors (#2388)
  Add error codes to site breakage reports (#2413)
  Don't set dryRun for alpha builds (#2412)
  Add VPN redemption retry event (#2409)
  Changes to hotfix process (#2406)
  Release/7.106.0-3 changes (#2411)
  Fix tab loading (#2410)
  Improve waitlist invite code checks (#2398)
  Reenable toggle on disallowing vpn (#2404)
  Bump BrowserServicesKit to 103.0.2 (#2393)
  ...

.github/workflows/pr.yml

@@ -128,6 +128,9 @@ jobs:
 
  name: Make Release Build
 
+ # Dependabot doesn't have access to all secrets, so we skip this job
+ if: github.actor != 'dependabot[bot]'
+
  runs-on: macos-13-xlarge
  timeout-minutes: 30
 

.github/workflows/release.yml

@@ -20,15 +20,13 @@ on:
  branches:
  - release/**
  - hotfix/**
- - coldfix/**
  - '!release/**-' # filter out PRs matching that pattern
  - '!hotfix/**-'
- - '!coldfix/**-'
  types: [closed]
 
 jobs:
  make-release:
- if: github.event.action == 0 || github.event.pull_request.merged == true # empty string returns 0; for case when workflow is triggered manually
+ if: github.event.action == 0 || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'Merge triggers release')) # empty string returns 0; for case when workflow is triggered manually
  runs-on: macos-13-xlarge
  name: Make App Store Connect Release
 
@@ -98,7 +96,7 @@ jobs:
 
  - name: Upload debug symbols to Asana
  if: ${{ always() && github.event.inputs.asana-task-url }}
- env:
+ env: 
  ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
  run: |
  if [[ -f ${{ env.dsyms_path }} ]]; then

.gitignore

@@ -74,3 +74,5 @@ fastlane/test_output
 
 Configuration/ExternalDeveloper.xcconfig
 scripts/assets
+
+DuckDuckGoTests/NetworkProtectionVPNLocationViewModelTests.swift*.plist

.maestro/release_tests/emailprotection.yaml

@@ -18,7 +18,9 @@ tags:
 - scroll
 - assertVisible: Email Protection
 - tapOn: Email Protection
+- assertVisible: Email privacy, simplified.
 - assertVisible:
  id: searchEntry
+- tapOn: 
+ id: "searchEntry"
 - assertVisible: https://duckduckgo.com/email/
-- assertVisible: Email privacy, simplified.

Configuration/Version.xcconfig

@@ -1 +1 @@
-MARKETING_VERSION = 7.106.0
+MARKETING_VERSION = 7.107.0

Core/AppPrivacyConfigurationDataProvider.swift

@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
  public struct Constants {
- public static let embeddedDataETag = "\"4e984b6034f1e27fe85fdad5f4bf37c9\""
- public static let embeddedDataSHA = "d599888e7b447bbaeb2d9a7fd7ccf06956fce8976c316be2f497561a6832613e"
+ public static let embeddedDataETag = "\"d0ae514c42e1e632584aba7a025b8b92\""
+ public static let embeddedDataSHA = "b304a2dbb2edc7443a4950bb2ba9f7604354cf32575dd5a9ca09acd5c4b78146"
  }
 
  public var embeddedDataEtag: String {

Core/CookieStorage.swift

@@ -22,16 +22,27 @@ import Foundation
 
 public class CookieStorage {
 
- struct Constants {
- static let key = "com.duckduckgo.allowedCookies"
+ struct Keys {
+ static let allowedCookies = "com.duckduckgo.allowedCookies"
+ static let consumed = "com.duckduckgo.consumedCookies"
  }
 
  private var userDefaults: UserDefaults
-
+
+ var isConsumed: Bool {
+ get {
+ userDefaults.bool(forKey: Keys.consumed, defaultValue: false)
+ }
+ set {
+ userDefaults.set(newValue, forKey: Keys.consumed)
+ }
+ }
+
+ /// Use the `updateCookies` function rather than the setter which is only visible for testing.
  var cookies: [HTTPCookie] {
  get {
  var storedCookies = [HTTPCookie]()
- if let cookies = userDefaults.object(forKey: Constants.key) as? [[String: Any?]] {
+ if let cookies = userDefaults.object(forKey: Keys.allowedCookies) as? [[String: Any?]] {
  for cookieData in cookies {
  var properties = [HTTPCookiePropertyKey: Any]()
  cookieData.forEach({
@@ -57,17 +68,76 @@ public class CookieStorage {
  }
  cookies.append(mappedCookie)
  }
- userDefaults.setValue(cookies, forKey: Constants.key)
+ userDefaults.setValue(cookies, forKey: Keys.allowedCookies)
  }
+
  }
 
  public init(userDefaults: UserDefaults = UserDefaults.app) {
  self.userDefaults = userDefaults
  }
 
- func clear() {
- userDefaults.removeObject(forKey: Constants.key)
- os_log("cleared cookies", log: .generalLog, type: .debug)
+ enum CookieDomainsOnUpdate {
+ case empty
+ case match
+ case missing
+ case different
  }
+
+ @discardableResult
+ func updateCookies(_ cookies: [HTTPCookie], keepingPreservedLogins preservedLogins: PreserveLogins) -> CookieDomainsOnUpdate {
+ isConsumed = false
+
+ let persisted = self.cookies
+
+ func cookiesByDomain(_ cookies: [HTTPCookie]) -> [String: [HTTPCookie]] {
+ var byDomain = [String: [HTTPCookie]]()
+ cookies.forEach { cookie in
+ var cookies = byDomain[cookie.domain, default: []]
+ cookies.append(cookie)
+ byDomain[cookie.domain] = cookies
+ }
+ return byDomain
+ }
+
+ let updatedCookiesByDomain = cookiesByDomain(cookies)
+ var persistedCookiesByDomain = cookiesByDomain(persisted)
 
+ // Do the diagnostics before the dicts get changed.
+ let diagnosticResult = evaluateDomains(
+ updatedDomains: updatedCookiesByDomain.keys.sorted(),
+ persistedDomains: persistedCookiesByDomain.keys.sorted()
+ )
+
+ updatedCookiesByDomain.keys.forEach {
+ persistedCookiesByDomain[$0] = updatedCookiesByDomain[$0]
+ }
+
+ persistedCookiesByDomain.keys.forEach {
+ guard $0 != "duckduckgo.com" else { return } // DDG cookies are for SERP settings only
+
+ if !preservedLogins.isAllowed(cookieDomain: $0) {
+ persistedCookiesByDomain.removeValue(forKey: $0)
+ }
+ }
+
+ let now = Date()
+ self.cookies = persistedCookiesByDomain.map { $0.value }.joined().compactMap { $0 }
+ .filter { $0.expiresDate == nil || $0.expiresDate! > now }
+
+ return diagnosticResult
+ }
+
+ private func evaluateDomains(updatedDomains: [String], persistedDomains: [String]) -> CookieDomainsOnUpdate {
+ if persistedDomains.isEmpty {
+ return .empty
+ } else if updatedDomains.count < persistedDomains.count {
+ return .missing
+ } else if updatedDomains == persistedDomains {
+ return .match
+ } else {
+ return .different
+ }
+ }
+
 }

Core/DailyPixel.swift

@@ -71,10 +71,16 @@ public final class DailyPixel {
  public static func fireDailyAndCount(pixel: Pixel.Event,
  error: Swift.Error? = nil,
  withAdditionalParameters params: [String: String] = [:],
+ includedParameters: [Pixel.QueryParameters] = [.atb, .appVersion],
  onDailyComplete: @escaping (Swift.Error?) -> Void = { _ in },
  onCountComplete: @escaping (Swift.Error?) -> Void = { _ in }) {
  if !pixel.hasBeenFiredToday(dailyPixelStorage: storage) {
- Pixel.fire(pixelNamed: pixel.name + "_d", withAdditionalParameters: params, onComplete: onDailyComplete)
+ Pixel.fire(
+ pixelNamed: pixel.name + "_d",
+ withAdditionalParameters: params,
+ includedParameters: includedParameters,
+ onComplete: onDailyComplete
+ )
  } else {
  onDailyComplete(Error.alreadyFired)
  }
@@ -83,7 +89,12 @@ public final class DailyPixel {
  if let error {
  newParams.appendErrorPixelParams(error: error)
  }
- Pixel.fire(pixelNamed: pixel.name + "_c", withAdditionalParameters: newParams, onComplete: onCountComplete)
+ Pixel.fire(
+ pixelNamed: pixel.name + "_c",
+ withAdditionalParameters: newParams,
+ includedParameters: includedParameters,
+ onComplete: onCountComplete
+ )
  }
 
  private static func updatePixelLastFireDate(pixel: Pixel.Event) {

Core/PixelEvent.swift

@@ -312,6 +312,7 @@ extension Pixel {
  case networkProtectionKeychainErrorFailedToCastKeychainValueToData
  case networkProtectionKeychainReadError
  case networkProtectionKeychainWriteError
+ case networkProtectionKeychainUpdateError
  case networkProtectionKeychainDeleteError
 
  case networkProtectionWireguardErrorCannotLocateTunnelFileDescriptor
@@ -341,6 +342,7 @@ extension Pixel {
  case networkProtectionWaitlistTermsAccepted
  case networkProtectionWaitlistNotificationShown
  case networkProtectionWaitlistNotificationLaunched
+ case networkProtectionWaitlistRetriedInviteCodeRedemption
 
  case networkProtectionGeoswitchingOpened
  case networkProtectionGeoswitchingSetNearest
@@ -521,6 +523,8 @@ extension Pixel {
  case emailIncontextModalExitEarlyContinue
 
  case compilationFailed
+
+ case appRatingPromptFetchError
  }
 
 }
@@ -810,6 +814,7 @@ extension Pixel.Event {
  case .networkProtectionKeychainErrorFailedToCastKeychainValueToData: return "m_netp_keychain_error_failed_to_cast_keychain_value_to_data"
  case .networkProtectionKeychainReadError: return "m_netp_keychain_error_read_failed"
  case .networkProtectionKeychainWriteError: return "m_netp_keychain_error_write_failed"
+ case .networkProtectionKeychainUpdateError: return "m_netp_keychain_error_update_failed"
  case .networkProtectionKeychainDeleteError: return "m_netp_keychain_error_delete_failed"
  case .networkProtectionWireguardErrorCannotLocateTunnelFileDescriptor: return "m_netp_wireguard_error_cannot_locate_tunnel_file_descriptor"
  case .networkProtectionWireguardErrorInvalidState: return "m_netp_wireguard_error_invalid_state"
@@ -833,6 +838,7 @@ extension Pixel.Event {
  case .networkProtectionWaitlistTermsAccepted: return "m_netp_waitlist_terms_accepted"
  case .networkProtectionWaitlistNotificationShown: return "m_netp_waitlist_notification_shown"
  case .networkProtectionWaitlistNotificationLaunched: return "m_netp_waitlist_notification_launched"
+ case .networkProtectionWaitlistRetriedInviteCodeRedemption: return "m_netp_waitlist_retried_invite_code_redemption"
 
  case .networkProtectionGeoswitchingOpened: return "m_netp_imp_geoswitching"
  case .networkProtectionGeoswitchingSetNearest: return "m_netp_ev_geoswitching_set_nearest"
@@ -1015,6 +1021,8 @@ extension Pixel.Event {
  // MARK: - Return user measurement
  case .debugReturnUserAddATB: return "m_debug_return_user_add_atb"
  case .debugReturnUserUpdateATB: return "m_debug_return_user_update_atb"
+
+ case .appRatingPromptFetchError: return "m_d_app_rating_prompt_fetch_error"
  }
 
  }

Core/WebCacheManager.swift

@@ -74,7 +74,7 @@ public class WebCacheManager {
 
  let cookies = cookieStorage.cookies
 
- guard !cookies.isEmpty else {
+ guard !cookies.isEmpty, !cookieStorage.isConsumed else {
  completion()
  return
  }
@@ -93,9 +93,9 @@ public class WebCacheManager {
 
  DispatchQueue.global(qos: .userInitiated).async {
  group.wait()
+ cookieStorage.isConsumed = true
 
  DispatchQueue.main.async {
- cookieStorage.clear()
  completion()
 
  if cookieStorage.cookies.count > 0 {
@@ -162,7 +162,7 @@ public class WebCacheManager {
  logins: PreserveLogins,
  storeIdManager: DataStoreIdManager,
  completion: @escaping () -> Void) {
-
+ 
  guard let containerId = storeIdManager.id else {
  completion()
  return
@@ -181,11 +181,8 @@ public class WebCacheManager {
  await checkForLeftBehindDataStores()
 
  storeIdManager.allocateNewContainerId()
- // If cookies is empty it's likely that the webview was not used since the last fire button so
- // don't overwrite previously saved cookies
- if let cookies, !cookies.isEmpty {
- cookieStorage.cookies = cookies
- }
+
+ cookieStorage.updateCookies(cookies ?? [], keepingPreservedLogins: logins)
 
  completion()
  }
@@ -208,7 +205,7 @@ public class WebCacheManager {
  // From this point onwards... use containers
  dataStoreIdManager.allocateNewContainerId()
  Task { @MainActor in
- cookieStorage.cookies = cookies
+ cookieStorage.updateCookies(cookies, keepingPreservedLogins: logins)
  completion()
  }
  } else {