From af7cbc1db4cd0169f1ba48b4031fc7644820e1e7 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Mon, 12 Feb 2024 13:56:53 -0700 Subject: [PATCH 01/10] Unify engines --- .../BVC+WKNavigationDelegate.swift | 4 +- .../BrowserViewController.swift | 2 +- .../Browser/Helpers/LaunchHelper.swift | 11 +- .../Brave/Frontend/Browser/PageData.swift | 12 +- .../FilterLists/FilterListsView.swift | 19 +- .../Paged/ContentBlockerScriptHandler.swift | 39 +- .../Paged/CosmeticFiltersScriptHandler.swift | 6 +- .../RequestBlockingContentScriptHandler.swift | 4 +- .../SiteStateListenerScriptHandler.swift | 4 +- .../AdBlock/AdBlockEngineManager.swift | 349 +++++++++++++++++ .../AdBlock/AdBlockGroupsManager.swift | 249 ++++++++++++ .../GroupedAdBlockEngine.swift} | 91 ++--- .../ContentBlockerManager.swift | 2 +- ...ctionPageStats.swift => TPPageStats.swift} | 47 +-- .../WebFilters/CustomFilterListStorage.swift | 2 +- .../Sources/Brave/WebFilters/FilterList.swift | 4 +- .../FilterListCustomURLDownloader.swift | 48 +-- .../FilterListResourceDownloader.swift | 137 +------ .../Brave/WebFilters/FilterListStorage.swift | 24 +- .../Adblock/AdBlockEngine+Extensions.swift | 2 +- .../ShieldStats/Adblock/AdBlockStats.swift | 355 ------------------ .../Tests/ClientTests/PageDataTests.swift | 14 +- ....swift => GroupedAdBlockEngineTests.swift} | 99 ++--- 23 files changed, 807 insertions(+), 717 deletions(-) create mode 100644 ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift create mode 100644 ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift rename ios/brave-ios/Sources/Brave/WebFilters/{ShieldStats/Adblock/CachedAdBlockEngine.swift => AdBlock/GroupedAdBlockEngine.swift} (73%) rename ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/{TrackingProtectionPageStats.swift => TPPageStats.swift} (51%) delete mode 100644 ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift rename ios/brave-ios/Tests/ClientTests/Web Filters/{CachedAdBlockEngineTests.swift => GroupedAdBlockEngineTests.swift} (69%) diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift index 3e07116d64c0..79c7fd95e4db 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift @@ -436,11 +436,11 @@ extension BrowserViewController: WKNavigationDelegate { { let domain = Domain.getOrCreate(forUrl: requestURL, persistent: !isPrivateBrowsing) - let shouldBlock = await AdBlockStats.shared.shouldBlock( + let shouldBlock = await AdBlockGroupsManager.shared.shouldBlock( requestURL: requestURL, sourceURL: requestURL, resourceType: .document, - isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + domain: domain ) if shouldBlock, let url = requestURL.encodeEmbeddedInternalURL(for: .blocked) { diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift index 64a036ed172e..feff1dec99c3 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift @@ -468,7 +468,7 @@ public class BrowserViewController: UIViewController { ScriptFactory.shared.clearCaches() Task { - await AdBlockStats.shared.didReceiveMemoryWarning() + await AdBlockGroupsManager.shared.didReceiveMemoryWarning() } for tab in tabManager.tabsForCurrentMode where tab.id != tabManager.selectedTab?.id { diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift index 8a8dcd5d9abb..f4e11c92f05a 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift @@ -153,9 +153,7 @@ extension FilterListStorage { // If we don't have filter lists yet loaded, use the settings return Set( allFilterListSettings.compactMap { setting -> ContentBlockerManager.BlocklistType? in - guard let componentId = setting.componentId else { return nil } - return .filterList( - componentId: componentId, + return setting.engineSource?.blocklistType( isAlwaysAggressive: setting.isAlwaysAggressive ) } @@ -163,10 +161,9 @@ extension FilterListStorage { } else { // If we do have filter lists yet loaded, use them as they are always the most up to date and accurate return Set( - filterLists.map { filterList in - return .filterList( - componentId: filterList.entry.componentId, - isAlwaysAggressive: filterList.isAlwaysAggressive + filterLists.compactMap { filterList in + return filterList.engineSource.blocklistType( + isAlwaysAggressive: filterList.engineType.isAlwaysAggressive ) } ) diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift index 1761ac07491e..004b3d9af1a7 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift @@ -22,11 +22,11 @@ struct PageData { /// These are loaded dyncamically as the user scrolls through the page private(set) var allSubframeURLs: Set = [] /// The stats class to get the engine data from - private var adBlockStats: AdBlockStats + private var groupsManager: AdBlockGroupsManager - init(mainFrameURL: URL, adBlockStats: AdBlockStats = AdBlockStats.shared) { + init(mainFrameURL: URL, groupsManager: AdBlockGroupsManager = AdBlockGroupsManager.shared) { self.mainFrameURL = mainFrameURL - self.adBlockStats = adBlockStats + self.groupsManager = groupsManager } /// This method builds all the user scripts that should be included for this page @@ -120,7 +120,7 @@ struct PageData { domain: Domain, isDeAmpEnabled: Bool ) async -> Set { - return await adBlockStats.makeEngineScriptTypes( + return await groupsManager.makeEngineScriptTypes( frameURL: mainFrameURL, isMainFrame: true, isDeAmpEnabled: isDeAmpEnabled, @@ -130,7 +130,7 @@ struct PageData { func makeAllEngineScripts(for domain: Domain, isDeAmpEnabled: Bool) async -> Set { // Add engine scripts for the main frame - async let engineScripts = adBlockStats.makeEngineScriptTypes( + async let engineScripts = groupsManager.makeEngineScriptTypes( frameURL: mainFrameURL, isMainFrame: true, isDeAmpEnabled: isDeAmpEnabled, @@ -139,7 +139,7 @@ struct PageData { // Add engine scripts for all of the known sub-frames async let additionalScriptTypes = allSubframeURLs.asyncConcurrentCompactMap({ frameURL in - return await self.adBlockStats.makeEngineScriptTypes( + return await self.groupsManager.makeEngineScriptTypes( frameURL: frameURL, isMainFrame: false, isDeAmpEnabled: isDeAmpEnabled, diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift index 023321469e6c..c6bc2ab2c034 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift @@ -16,9 +16,6 @@ struct FilterListsView: View { @ObservedObject private var customFilterListStorage = CustomFilterListStorage.shared @Environment(\.editMode) private var editMode @State private var showingAddSheet = false - @State private var expectedEnabledSources: Set = Set( - AdBlockStats.shared.enabledSources - ) private let dateFormatter = RelativeDateTimeFormatter() var body: some View { @@ -68,8 +65,7 @@ struct FilterListsView: View { } .onDisappear { Task.detached { - await AdBlockStats.shared.removeDisabledEngines() - await AdBlockStats.shared.ensureEnabledEngines() + await AdBlockGroupsManager.shared.ensureEnabledEngines() } } } @@ -86,13 +82,6 @@ struct FilterListsView: View { .foregroundColor(Color(.secondaryBraveLabel)) } } - .onChange(of: filterList.isEnabled) { isEnabled in - if isEnabled { - expectedEnabledSources.insert(filterList.engineSource) - } else { - expectedEnabledSources.remove(filterList.engineSource) - } - } } } } @@ -129,12 +118,6 @@ struct FilterListsView: View { } } .onChange(of: filterListURL.setting.isEnabled) { isEnabled in - if isEnabled { - expectedEnabledSources.insert(filterListURL.setting.engineSource) - } else { - expectedEnabledSources.remove(filterListURL.setting.engineSource) - } - Task { CustomFilterListSetting.save(inMemory: !customFilterListStorage.persistChanges) } diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift index 4081beca165e..0b4d8a5e1b43 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift @@ -22,6 +22,11 @@ extension ContentBlockerHelper: TabContentScript { let data: [ContentblockerDTOData] } + enum BlockedType: Hashable { + case image + case ad + } + static let scriptName = "TrackingProtectionStats" static let scriptId = UUID().uuidString static let messageHandlerName = "\(scriptName)_\(messageUUID)" @@ -94,12 +99,12 @@ extension ContentBlockerHelper: TabContentScript { guard let domainURLString = domain.url else { return } let genericTypes = ContentBlockerManager.shared.validGenericTypes(for: domain) - let blockedType = await TPStatsBlocklistChecker.shared.blockedTypes( + let blockedType = await blockedTypes( requestURL: requestURL, sourceURL: sourceURL, enabledRuleTypes: genericTypes, resourceType: dto.resourceType, - isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + domain: domain ) guard let blockedType = blockedType else { return } @@ -148,4 +153,34 @@ extension ContentBlockerHelper: TabContentScript { Logger.module.error("\(error.localizedDescription)") } } + + @MainActor func blockedTypes( + requestURL: URL, + sourceURL: URL, + enabledRuleTypes: Set, + resourceType: AdblockEngine.ResourceType, + domain: Domain + ) async -> BlockedType? { + guard let host = requestURL.host, !host.isEmpty else { + // TP Stats init isn't complete yet + return nil + } + + if resourceType == .image && Preferences.Shields.blockImages.value { + return .image + } + + if enabledRuleTypes.contains(.blockAds) || enabledRuleTypes.contains(.blockTrackers) { + if await AdBlockGroupsManager.shared.shouldBlock( + requestURL: requestURL, + sourceURL: sourceURL, + resourceType: resourceType, + domain: domain + ) { + return .ad + } + } + + return nil + } } diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift index c14b994e2b84..c00ede362823 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift @@ -61,9 +61,9 @@ class CosmeticFiltersScriptHandler: TabContentScript { forUrl: frameURL, persistent: self.tab?.isPrivate == true ? false : true ) - let cachedEngines = await AdBlockStats.shared.cachedEngines(for: domain) + let cachedEngines = AdBlockGroupsManager.shared.cachedEngines(for: domain) - let selectorArrays = await cachedEngines.asyncConcurrentCompactMap { + let selectorArrays = await cachedEngines.asyncCompactMap { cachedEngine -> (selectors: Set, isAlwaysAggressive: Bool)? in do { guard @@ -76,7 +76,7 @@ class CosmeticFiltersScriptHandler: TabContentScript { return nil } - return (selectors, cachedEngine.isAlwaysAggressive) + return (selectors, cachedEngine.type.isAlwaysAggressive) } catch { Logger.module.error("\(error.localizedDescription)") return nil diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift index 9726d8744cc1..1194b5a72e84 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift @@ -79,11 +79,11 @@ class RequestBlockingContentScriptHandler: TabContentScript { Task { @MainActor in let domain = Domain.getOrCreate(forUrl: currentTabURL, persistent: !isPrivateBrowsing) guard let domainURLString = domain.url else { return } - let shouldBlock = await AdBlockStats.shared.shouldBlock( + let shouldBlock = await AdBlockGroupsManager.shared.shouldBlock( requestURL: requestURL, sourceURL: sourceURL, resourceType: dto.data.resourceType, - isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + domain: domain ) // Ensure we check that the stats we're tracking is still for the same page diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift index f951989b87b7..55b67b36ea94 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift @@ -77,7 +77,7 @@ class SiteStateListenerScriptHandler: TabContentScript { return } - let models = await AdBlockStats.shared.cosmeticFilterModels( + let models = await AdBlockGroupsManager.shared.cosmeticFilterModels( forFrameURL: frameURL, domain: domain ) @@ -102,7 +102,7 @@ class SiteStateListenerScriptHandler: TabContentScript { } @MainActor private func makeSetup( - from modelTuples: [AdBlockStats.CosmeticFilterModelTuple], + from modelTuples: [AdBlockGroupsManager.CosmeticFilterModelTuple], isAggressive: Bool ) throws -> UserScriptType.SelectorsPollerSetup { var standardSelectors: Set = [] diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift new file mode 100644 index 000000000000..7ec5c5a6a37f --- /dev/null +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -0,0 +1,349 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Data +import Foundation +import Preferences +import os + +/// A class for managing a grouped engine +@MainActor class AdBlockEngineManager { + /// The directory to which we should store the unified list into + private static var cacheFolderDirectory: FileManager.SearchPathDirectory { + return FileManager.SearchPathDirectory.applicationSupportDirectory + } + + /// The top level folder to create for caching engine data + private static let parentCacheFolderName = "engines" + /// All the info that is currently available + private var availableInfos: [GroupedAdBlockEngine.FilterListInfo] + /// The subfolder that the cache data is stored. + /// Since we can have multiple engines this folder will be unique per engine + private var cacheFolderName: String + /// The type of engine this represents what kind of blocking we are doing. (aggressive or standard) + /// The difference is in how we treat first party blocking. Aggressive also blocks first party content. + let engineType: GroupedAdBlockEngine.EngineType + private(set) var engine: GroupedAdBlockEngine? + + /// This structure represents encodable info on what cached engine data contains + private struct CachedEngineInfo: Codable { + let infos: [GroupedAdBlockEngine.FilterListInfo] + let fileType: GroupedAdBlockEngine.FileType + } + + /// Get the already created cache folder + /// + /// - Note: Returns nil if the cache folder does not exist + private var createdCacheFolderURL: URL? { + guard let folderURL = Self.cacheFolderDirectory.url else { return nil } + let cacheFolderURL = folderURL.appendingPathComponent( + Self.parentCacheFolderName, + conformingTo: .folder + ) + .appendingPathComponent(cacheFolderName, conformingTo: .folder) + + if FileManager.default.fileExists(atPath: cacheFolderURL.path) { + return cacheFolderURL + } else { + return nil + } + } + + /// Get the url of the cached file + /// + /// - Note: Returns nil if the file does not exist + private var savedCombinedListURL: URL? { + guard let cacheFolderURL = createdCacheFolderURL else { + return nil + } + + let fileURL = cacheFolderURL.appendingPathComponent("list.txt", conformingTo: .text) + + if FileManager.default.fileExists(atPath: fileURL.path) { + return fileURL + } else { + return nil + } + } + + /// Return an array of all sources that are enabled according to user's settings + /// - Note: This does not take into account the domain or global adblock toggle + private var enabledSources: [GroupedAdBlockEngine.Source] { + var enabledSources = FilterListStorage.shared.enabledSources(engineType: engineType) + if engineType.isAlwaysAggressive { + enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) + } + return enabledSources + } + + /// All the infos that are compilable based on the enabled sources and available infos + var compilableInfos: [GroupedAdBlockEngine.FilterListInfo] { + return enabledSources.compactMap { source in + return availableInfos.first(where: { $0.source == source }) + } + } + + init(engineType: GroupedAdBlockEngine.EngineType, cacheFolderName: String) { + self.availableInfos = [] + self.engine = nil + self.engineType = engineType + self.cacheFolderName = cacheFolderName + } + + /// Tells us if this source should be loaded. + func isEnabled(source: GroupedAdBlockEngine.Source) -> Bool { + return enabledSources.contains(source) + } + + /// Add the info to the available list + func add(info: GroupedAdBlockEngine.FilterListInfo) { + availableInfos.removeAll { exisitingInfo in + return exisitingInfo.source == info.source && exisitingInfo.version <= info.version + } + + availableInfos.append(info) + } + + /// Remove any info from the available list given by the source + /// Mostly used for custom filter lists + func removeInfo(for source: GroupedAdBlockEngine.Source) { + availableInfos.removeAll { exisitingInfo in + return exisitingInfo.source == source + } + } + + /// Checks to see if we need to compile or recompile the engine based on the available info + func checkNeedsCompile() -> Bool { + let compilableInfos = compilableInfos + guard !compilableInfos.isEmpty else { return false } + return compilableInfos != engine?.group.infos + } + + /// Load the engine from cache so it can be ready during launch + func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async throws { + guard let localFileURL = savedCombinedListURL else { return } + guard let cacheInfo = loadCachedInfo() else { return } + + let groupedEngine = try GroupedAdBlockEngine.compile( + group: GroupedAdBlockEngine.FilterListGroup( + infos: cacheInfo.infos, + localFileURL: localFileURL, + fileType: cacheInfo.fileType + ), + type: engineType + ) + + if let resourcesInfo = resourcesInfo { + try await groupedEngine.useResources(from: resourcesInfo) + } + + self.set(engine: groupedEngine) + } + + /// This is a task that we use to delay a requested compile. + /// This allows us to wait for more sources and limit the number of times we compile + private var delayTask: Task? + /// This is the current pending group we are compiling. + /// This allows us to ensure we are not compiling a newer version of the rules + private var pendingGroup: GroupedAdBlockEngine.FilterListGroup? + + /// This will compile available data, but will wait a little bit in case something new gets downloaded. + /// Especially needed during launch when we have a bunch of downloads coming at the same time. + func compileDelayed(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) { + // Cancel the previous task + delayTask?.cancel() + + // Restart the task + delayTask = Task { + try await Task.sleep(seconds: 10) + try await compileAvailable(resourcesInfo: resourcesInfo) + } + } + + /// Compile an engine from all available data + func compileAvailable(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async throws { + // 1. Create the group + let group = try combineRules() + self.pendingGroup = group + + // 2. Compile the engine + let groupedEngine = try GroupedAdBlockEngine.compile(group: group, type: engineType) + if let resourcesInfo { + try await groupedEngine.useResources(from: resourcesInfo) + } + + // 3. Ensure our file is still up to date before setting it + // (avoid race conditiions) + guard pendingGroup == group else { return } + self.set(engine: groupedEngine) + saveCachedInfo(from: groupedEngine) + self.pendingGroup = nil + } + + private func set(engine: GroupedAdBlockEngine) { + let infosString = engine.group.infos.map({ " \($0.debugDescription)" }).joined(separator: "\n") + ContentBlockerManager.log.debug( + "Set `\(self.cacheFolderName)` engine from \(engine.group.infos.count) sources:\n\(infosString)" + ) + self.engine = engine + } + + /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. + func update(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) async throws { + try await engine?.useResources(from: resourcesInfo) + } + + func ensureContentBlockers(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) async { + guard + let blocklistType = filterListInfo.source.blocklistType( + isAlwaysAggressive: engineType.isAlwaysAggressive + ) + else { + return + } + + let modes = await ContentBlockerManager.shared.missingModes(for: blocklistType) + guard !modes.isEmpty else { return } + + do { + try await ContentBlockerManager.shared.compileRuleList( + at: filterListInfo.localFileURL, + for: blocklistType, + modes: modes + ) + } catch { + ContentBlockerManager.log.error( + "Failed to compile rule list for \(filterListInfo.debugDescription)" + ) + } + } + + /// Take all the filter lists and combine them into one then save them into a cache folder. + private func combineRules() throws -> GroupedAdBlockEngine.FilterListGroup { + // 1. Grab all the needed infos + let infos = compilableInfos + + // 2. Create a file url + let cachedFolder = try getOrCreateCacheFolder() + let fileURL = cachedFolder.appendingPathComponent("list.txt", conformingTo: .text) + + // 3. Join all the rules together + let unifiedRules = infos.compactMap { info in + do { + return try String(contentsOf: info.localFileURL) + } catch { + ContentBlockerManager.log.error( + "Could not load rules for `\(info.debugDescription)`: \(error)" + ) + return nil + } + }.joined(separator: "\n") + + // 4. Save the files into storage + if FileManager.default.fileExists(atPath: fileURL.path) { + try FileManager.default.removeItem(at: fileURL) + } + try unifiedRules.write(to: fileURL, atomically: true, encoding: .utf8) + + // 4. Return a group containing info on this new file + return GroupedAdBlockEngine.FilterListGroup( + infos: infos, + localFileURL: fileURL, + fileType: .text + ) + } + + /// Get or create a cache folder for the given `Resource` + /// + /// - Note: This technically can't really return nil as the location and folder are hard coded + private func getOrCreateCacheFolder() throws -> URL { + guard + let folderURL = FileManager.default.getOrCreateFolder( + name: [Self.parentCacheFolderName, cacheFolderName].joined(separator: "/"), + location: Self.cacheFolderDirectory + ) + else { + throw ResourceFileError.failedToCreateCacheFolder + } + + return folderURL + } + + private func saveCachedInfo(from engine: GroupedAdBlockEngine) { + let encoder = JSONEncoder() + let info = CachedEngineInfo(infos: engine.group.infos, fileType: engine.group.fileType) + + do { + let data = try encoder.encode(info) + let folderURL = try getOrCreateCacheFolder() + let fileURL = folderURL.appendingPathComponent("engine_info", conformingTo: .json) + + if FileManager.default.fileExists(atPath: fileURL.path) { + try FileManager.default.removeItem(at: fileURL) + } + + try data.write(to: fileURL) + } catch { + ContentBlockerManager.log.error( + "Failed to save cache info for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + } + } + + private func loadCachedInfo() -> CachedEngineInfo? { + let decoder = JSONDecoder() + + do { + let folderURL = try getOrCreateCacheFolder() + let fileURL = folderURL.appendingPathComponent("engine_info", conformingTo: .json) + guard FileManager.default.fileExists(atPath: fileURL.path) else { return nil } + let data = try Data(contentsOf: fileURL) + return try decoder.decode(CachedEngineInfo.self, from: data) + } catch { + ContentBlockerManager.log.error( + "Failed to load cache info for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + return nil + } + } +} + +extension GroupedAdBlockEngine.Source { + func blocklistType(isAlwaysAggressive: Bool) -> ContentBlockerManager.BlocklistType? { + switch self { + case .filterList(let componentId, let uuid): + guard uuid != FilterList.defaultComponentUUID else { + // For now we don't compile this into content blockers because we use the one coming from slim list + // We might change this in the future as it ends up with 95k items whereas the limit is 150k. + // So there is really no reason to use slim list except perhaps for performance which we need to test out. + return nil + } + + return .filterList(componentId: componentId, isAlwaysAggressive: isAlwaysAggressive) + case .filterListURL(let uuid): + return .customFilterList(uuid: uuid) + } + } +} + +extension FilterListSetting { + @MainActor var engineSource: GroupedAdBlockEngine.Source? { + guard let componentId = componentId else { return nil } + return .filterList(componentId: componentId, uuid: uuid) + } +} + +extension FilterList { + @MainActor var engineSource: GroupedAdBlockEngine.Source { + return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) + } +} + +extension CustomFilterListSetting { + @MainActor var engineSource: GroupedAdBlockEngine.Source { + return .filterListURL(uuid: uuid) + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift new file mode 100644 index 000000000000..933118ec9804 --- /dev/null +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -0,0 +1,249 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Data +import Foundation +import Preferences +import os + +/// This object holds on to our adblock engines and returns information needed for stats tracking as well as some conveniences +/// for injected scripts needed during web navigation and cosmetic filters models needed by the `SelectorsPollerScript.js` script. +@MainActor public class AdBlockGroupsManager { + typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) + public static let shared = AdBlockGroupsManager( + standardManager: GroupedAdBlockEngine.EngineType.standard.makeDefaultManager(), + aggressiveManager: GroupedAdBlockEngine.EngineType.aggressive.makeDefaultManager() + ) + + private let standardManager: AdBlockEngineManager + private let aggressiveManager: AdBlockEngineManager + + private var allManagers: [AdBlockEngineManager] { + return [standardManager, aggressiveManager] + } + + /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. + public var resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + + init(standardManager: AdBlockEngineManager, aggressiveManager: AdBlockEngineManager) { + self.standardManager = standardManager + self.aggressiveManager = aggressiveManager + self.resourcesInfo = nil + } + + /// Handle memory warnings by freeing up some memory + func didReceiveMemoryWarning() async { + await standardManager.engine?.clearCaches() + await aggressiveManager.engine?.clearCaches() + } + + /// Load any cache data so its ready right during launch + func loadFromCache() async { + if let resourcesFolderURL = FilterListSetting.makeFolderURL( + forComponentFolderPath: Preferences.AppState.lastAdBlockResourcesFolderPath.value + ), FileManager.default.fileExists(atPath: resourcesFolderURL.path), + let resourcesInfo = getResourcesInfo(fromFolderURL: resourcesFolderURL) + { + // We need this for all filter lists so we can't compile anything until we download it + self.resourcesInfo = resourcesInfo + } + + await allManagers.asyncConcurrentForEach { manager in + guard manager.engineType.loadFromCache else { return } + + do { + try await manager.loadFromCache(resourcesInfo: self.resourcesInfo) + } catch { + ContentBlockerManager.log.error("Failed to load engine from cache: \(error)") + } + } + } + + /// Inform this manager of updates to the resources so our engines can be updated + func didUpdateResourcesComponent(folderURL: URL) async { + await Task { @MainActor in + let folderSubPath = FilterListSetting.extractFolderPath(fromComponentFolderURL: folderURL) + Preferences.AppState.lastAdBlockResourcesFolderPath.value = folderSubPath + }.value + + let version = folderURL.lastPathComponent + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: folderURL.appendingPathComponent("resources.json", conformingTo: .json), + version: version + ) + + updateIfNeeded(resourcesInfo: resourcesInfo) + } + + /// Handle updated filter list info + func updated( + filterListInfo: GroupedAdBlockEngine.FilterListInfo, + engineType: GroupedAdBlockEngine.EngineType + ) async { + let manager = getManager(for: engineType) + // Always update the info on the manager + manager.add(info: filterListInfo) + + if manager.checkNeedsCompile() { + manager.compileDelayed(resourcesInfo: resourcesInfo) + } + + // Compile content blockers if this filter list is enabled + if manager.isEnabled(source: filterListInfo.source) { + await manager.ensureContentBlockers(for: filterListInfo) + } + } + + /// Ensure all engines and content blockers are compiled + func ensureEnabledEngines() async { + guard let resourcesInfo = self.resourcesInfo else { return } + + await allManagers.asyncConcurrentForEach { manager in + // Compile engines + do { + if manager.checkNeedsCompile() { + try await manager.compileAvailable(resourcesInfo: resourcesInfo) + } + } catch { + // Ignore cancellation errors + } + + // Compile all content blockers for the given manager + await manager.compilableInfos.asyncForEach { filterListInfo in + await manager.ensureContentBlockers(for: filterListInfo) + } + } + } + + private func getResourcesInfo(fromFolderURL folderURL: URL) -> GroupedAdBlockEngine.ResourcesInfo? + { + let version = folderURL.lastPathComponent + return GroupedAdBlockEngine.ResourcesInfo( + localFileURL: folderURL.appendingPathComponent("resources.json", conformingTo: .json), + version: version + ) + } + + /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. + private func updateIfNeeded(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) { + guard self.resourcesInfo == nil || resourcesInfo.version > self.resourcesInfo!.version else { + return + } + self.resourcesInfo = resourcesInfo + + allManagers.forEach { manager in + Task { + try await manager.update(resourcesInfo: resourcesInfo) + } + } + + if #available(iOS 16.0, *) { + ContentBlockerManager.log.debug( + "Updated resources component: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" + ) + } + } + + /// Checks the general and regional engines to see if the request should be blocked + func shouldBlock( + requestURL: URL, + sourceURL: URL, + resourceType: AdblockEngine.ResourceType, + domain: Domain + ) async -> Bool { + return await cachedEngines(for: domain).asyncConcurrentMap({ cachedEngine in + return await cachedEngine.shouldBlock( + requestURL: requestURL, + sourceURL: sourceURL, + resourceType: resourceType, + isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + ) + }).contains(where: { $0 }) + } + + /// This returns all the user script types for the given frame + func makeEngineScriptTypes( + frameURL: URL, + isMainFrame: Bool, + isDeAmpEnabled: Bool, + domain: Domain + ) async -> Set { + // Add any engine scripts for this frame + return await cachedEngines(for: domain).enumerated().asyncMap({ + index, + cachedEngine -> Set in + do { + return try await cachedEngine.makeEngineScriptTypes( + frameURL: frameURL, + isMainFrame: isMainFrame, + domain: domain, + isDeAmpEnabled: isDeAmpEnabled, + index: index + ) + } catch { + assertionFailure() + return [] + } + }).reduce( + Set(), + { partialResult, scriptTypes in + return partialResult.union(scriptTypes) + } + ) + } + + /// Returns all appropriate engines for the given domain + func cachedEngines(for domain: Domain) -> [GroupedAdBlockEngine] { + guard domain.isShieldExpected(.adblockAndTp, considerAllShieldsOption: true) else { return [] } + return allManagers.compactMap({ $0.engine }) + } + + /// Returns all the models for this frame URL + func cosmeticFilterModels( + forFrameURL frameURL: URL, + domain: Domain + ) async -> [CosmeticFilterModelTuple] { + return await cachedEngines(for: domain).asyncConcurrentCompactMap { + cachedEngine -> CosmeticFilterModelTuple? in + do { + guard let model = try await cachedEngine.cosmeticFilterModel(forFrameURL: frameURL) else { + return nil + } + return (cachedEngine.type.isAlwaysAggressive, model) + } catch { + assertionFailure() + return nil + } + } + } + + private func getManager(for engineType: GroupedAdBlockEngine.EngineType) -> AdBlockEngineManager { + switch engineType { + case .standard: return standardManager + case .aggressive: return aggressiveManager + } + } +} + +extension GroupedAdBlockEngine.EngineType { + fileprivate var defaultCachedFolderName: String { + switch self { + case .standard: return "standard" + case .aggressive: return "aggressive" + } + } + + fileprivate var loadFromCache: Bool { + switch self { + case .standard: return true + case .aggressive: return false + } + } + + @MainActor fileprivate func makeDefaultManager() -> AdBlockEngineManager { + return AdBlockEngineManager(engineType: self, cacheFolderName: defaultCachedFolderName) + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift similarity index 73% rename from ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift rename to ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift index 1e0c87f0f9d5..9d4da7edd68a 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift @@ -1,4 +1,4 @@ -// Copyright 2022 The Brave Authors. All rights reserved. +// Copyright 2024 The Brave Authors. All rights reserved. // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -11,8 +11,8 @@ import os /// An object that wraps around an `AdblockEngine` and caches some results /// and ensures information is always returned on the correct thread on the engine. -public actor CachedAdBlockEngine { - public enum Source: Hashable, CustomDebugStringConvertible { +public actor GroupedAdBlockEngine { + public enum Source: Codable, Hashable, CustomDebugStringConvertible { case filterList(componentId: String, uuid: String) case filterListURL(uuid: String) @@ -24,7 +24,7 @@ public actor CachedAdBlockEngine { } } - public enum FileType: Hashable, CustomDebugStringConvertible { + public enum FileType: Codable, Hashable, CustomDebugStringConvertible { case text, data public var debugDescription: String { @@ -35,14 +35,35 @@ public actor CachedAdBlockEngine { } } - public struct FilterListInfo: Hashable, Equatable, CustomDebugStringConvertible { - let source: Source + public enum EngineType: Hashable, CaseIterable { + case standard + case aggressive + + var isAlwaysAggressive: Bool { + switch self { + case .standard: return false + case .aggressive: return true + } + } + } + + public struct FilterListInfo: Codable, Hashable, Equatable, CustomDebugStringConvertible { + let source: GroupedAdBlockEngine.Source let localFileURL: URL let version: String - let fileType: FileType public var debugDescription: String { - return "`\(source.debugDescription)` v\(version) (\(fileType.debugDescription))" + return "`\(source.debugDescription)` v\(version)" + } + } + + public struct FilterListGroup: Hashable, Equatable, CustomDebugStringConvertible { + let infos: [FilterListInfo] + let localFileURL: URL + let fileType: GroupedAdBlockEngine.FileType + + public var debugDescription: String { + return infos.map({ $0.debugDescription }).joined(separator: ", ") } } @@ -61,20 +82,14 @@ public actor CachedAdBlockEngine { private let engine: AdblockEngine - let isAlwaysAggressive: Bool - let filterListInfo: FilterListInfo - let resourcesInfo: ResourcesInfo + let type: EngineType + let group: FilterListGroup + private(set) var resourcesInfo: ResourcesInfo? - init( - engine: AdblockEngine, - filterListInfo: FilterListInfo, - resourcesInfo: ResourcesInfo, - isAlwaysAggressive: Bool - ) { + init(engine: AdblockEngine, group: FilterListGroup, type: EngineType) { self.engine = engine - self.filterListInfo = filterListInfo - self.resourcesInfo = resourcesInfo - self.isAlwaysAggressive = isAlwaysAggressive + self.group = group + self.type = type } /// Return the selectors that need to be hidden given the frameURL, ids and classes @@ -129,7 +144,7 @@ public actor CachedAdBlockEngine { requestURL: requestURL, sourceURL: sourceURL, resourceType: resourceType, - isAggressive: isAggressiveMode || self.isAlwaysAggressive + isAggressive: isAggressiveMode || self.type.isAlwaysAggressive ) cachedShouldBlockResult.addElement(shouldBlock, forKey: key) @@ -175,47 +190,39 @@ public actor CachedAdBlockEngine { cachedFrameScriptTypes = FifoDict() } - /// Serialize the engine into data to be later loaded from cache - public func serialize() throws -> Data { - return try engine.serialize() + func useResources(from info: ResourcesInfo) throws { + try engine.useResources(fromFileURL: info.localFileURL) + resourcesInfo = info } /// Create an engine from the given resources public static func compile( - filterListInfo: FilterListInfo, - resourcesInfo: ResourcesInfo, - isAlwaysAggressive: Bool - ) throws -> CachedAdBlockEngine { + group: FilterListGroup, + type: EngineType + ) throws -> GroupedAdBlockEngine { let signpostID = Self.signpost.makeSignpostID() let state = Self.signpost.beginInterval( "compileEngine", id: signpostID, - "\(filterListInfo.debugDescription)" + "\(group.debugDescription)" ) do { - let engine = try makeEngine(from: filterListInfo) - try engine.useResources(fromFileURL: resourcesInfo.localFileURL) + let engine = try makeEngine(from: group) Self.signpost.endInterval("compileEngine", state) - - return CachedAdBlockEngine( - engine: engine, - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: isAlwaysAggressive - ) + return GroupedAdBlockEngine(engine: engine, group: group, type: type) } catch { Self.signpost.endInterval("compileEngine", state, "\(error.localizedDescription)") throw error } } - private static func makeEngine(from filterListInfo: FilterListInfo) throws -> AdblockEngine { - switch filterListInfo.fileType { + private static func makeEngine(from group: FilterListGroup) throws -> AdblockEngine { + switch group.fileType { case .data: - return try AdblockEngine(serializedData: Data(contentsOf: filterListInfo.localFileURL)) + return try AdblockEngine(serializedData: Data(contentsOf: group.localFileURL)) case .text: - return try AdblockEngine(rules: String(contentsOf: filterListInfo.localFileURL)) + return try AdblockEngine(rules: String(contentsOf: group.localFileURL)) } } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift index dba980798480..1c22a83f62b8 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift @@ -435,7 +435,7 @@ actor ContentBlockerManager { guard filterList.isEnabled else { return nil } return .filterList( componentId: filterList.entry.componentId, - isAlwaysAggressive: filterList.isAlwaysAggressive + isAlwaysAggressive: filterList.engineType.isAlwaysAggressive ) } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TPPageStats.swift similarity index 51% rename from ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift rename to ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TPPageStats.swift index a104edf7fe87..ea0d2f15faef 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TPPageStats.swift @@ -1,15 +1,9 @@ +// Copyright 2024 The Brave Authors. All rights reserved. // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// This file is largely verbatim from Focus iOS (Blockzilla/Lib/TrackingProtection). -// The preload and postload js files are unmodified from Focus. - -import BraveCore -import Data import Foundation -import Preferences -import Shared struct TPPageStats { let adCount: Int @@ -50,42 +44,3 @@ struct TPPageStats { ) } } - -class TPStatsBlocklistChecker { - static let shared = TPStatsBlocklistChecker() - - enum BlockedType: Hashable { - case image - case ad - } - - @MainActor func blockedTypes( - requestURL: URL, - sourceURL: URL, - enabledRuleTypes: Set, - resourceType: AdblockEngine.ResourceType, - isAggressiveMode: Bool - ) async -> BlockedType? { - guard let host = requestURL.host, !host.isEmpty else { - // TP Stats init isn't complete yet - return nil - } - - if resourceType == .image && Preferences.Shields.blockImages.value { - return .image - } - - if enabledRuleTypes.contains(.blockAds) || enabledRuleTypes.contains(.blockTrackers) { - if await AdBlockStats.shared.shouldBlock( - requestURL: requestURL, - sourceURL: sourceURL, - resourceType: resourceType, - isAggressiveMode: isAggressiveMode - ) { - return .ad - } - } - - return nil - } -} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift b/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift index b8a097125629..70d42cf3a3d4 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift @@ -67,7 +67,7 @@ import Foundation extension CustomFilterListStorage { /// Gives us source representations of all the enabled custom filter lists - @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + @MainActor var enabledSources: [GroupedAdBlockEngine.Source] { return filterListsURLs .filter(\.setting.isEnabled) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift index e06be2fab019..145d99b628d6 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift @@ -33,8 +33,8 @@ struct FilterList: Identifiable { /// Lets us know if this filter list is always aggressive. /// This value comes from `list_catalog.json` in brave core - var isAlwaysAggressive: Bool { - return !entry.firstPartyProtections + var engineType: GroupedAdBlockEngine.EngineType { + return entry.firstPartyProtections ? .standard : .aggressive } init(from entry: AdblockFilterListCatalogEntry, order: Int, isEnabled: Bool?) { diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift index 812e23e7c44f..b31b7e0698aa 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift @@ -63,55 +63,23 @@ actor FilterListCustomURLDownloader: ObservableObject { } } - /// Handle the download results of a custom filter list. This will process the download by compiling iOS rule lists and adding the rule list to the `AdblockEngineManager`. + /// Handle the download results of a custom filter list. This will process the download by compiling iOS rule lists and adding the rule list to the `AdBlockEngineManager`. private func handle( downloadResult: ResourceDownloader.DownloadResult, for filterListCustomURL: FilterListCustomURL ) async { - let uuid = await filterListCustomURL.setting.uuid - - // Add/remove the resource depending on if it is enabled/disabled - guard let resourcesInfo = await AdBlockStats.shared.resourcesInfo else { - assertionFailure("This should not have been called if the resources are not ready") - return - } - let source = await filterListCustomURL.setting.engineSource let version = fileVersionDateFormatter.string(from: downloadResult.date) - let filterListInfo = CachedAdBlockEngine.FilterListInfo( - source: .filterListURL(uuid: uuid), + + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( + source: source, localFileURL: downloadResult.fileURL, - version: version, - fileType: .text - ) - let lazyInfo = AdBlockStats.LazyFilterListInfo( - filterListInfo: filterListInfo, - isAlwaysAggressive: true + version: version ) - guard await AdBlockStats.shared.isEnabled(source: source) else { - await AdBlockStats.shared.updateIfNeeded( - filterListInfo: filterListInfo, - isAlwaysAggressive: true - ) - - // To free some space, remove any rule lists that are not needed - if let blocklistType = lazyInfo.blocklistType { - do { - try await ContentBlockerManager.shared.removeRuleLists(for: blocklistType) - } catch { - ContentBlockerManager.log.error( - "Failed to remove rule lists for \(filterListInfo.debugDescription)" - ) - } - } - return - } - - await AdBlockStats.shared.compile( - lazyInfo: lazyInfo, - resourcesInfo: resourcesInfo, - compileContentBlockers: true + await AdBlockGroupsManager.shared.updated( + filterListInfo: filterListInfo, + engineType: .aggressive ) } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index 30870a884e34..587669af33cd 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -38,60 +38,8 @@ public actor FilterListResourceDownloader { /// - Warning: This method loads filter list settings. /// You need to wait for `DataController.shared.initializeOnce()` to be called first before invoking this method public func loadFilterListSettingsAndCachedData() async { - guard - let resourcesFolderURL = await FilterListSetting.makeFolderURL( - forComponentFolderPath: Preferences.AppState.lastAdBlockResourcesFolderPath.value - ), FileManager.default.fileExists(atPath: resourcesFolderURL.path) - else { - // We need this for all filter lists so we can't compile anything until we download it - return - } - - let resourcesInfo = await didUpdateResourcesComponent(folderURL: resourcesFolderURL) - await compileCachedFilterLists(resourcesInfo: resourcesInfo) - } - - /// This function adds engine resources to `AdBlockManager` from cached data representing the enabled filter lists. - /// - /// The filter lists are additional blocking content that can be added to the`AdBlockEngine` and as iOS content blockers. - /// It represents the items found in the "Filter lists" section of the "Shields" menu. - /// - /// - Note: The content blockers for these filter lists are not loaded at this point. They are cached by iOS and there is no need to reload them. - /// - /// - Warning: This method loads filter list settings. - /// You need to wait for `DataController.shared.initializeOnce()` to be called first before invoking this method - private func compileCachedFilterLists(resourcesInfo: CachedAdBlockEngine.ResourcesInfo) async { await FilterListStorage.shared.loadFilterListSettings() - let filterListSettings = await FilterListStorage.shared.allFilterListSettings - - do { - try await filterListSettings.asyncConcurrentForEach { setting in - guard await setting.isEagerlyLoaded == true else { return } - guard let componentId = await setting.componentId else { return } - guard !FilterList.disabledComponentIDs.contains(componentId) else { return } - guard let source = await setting.engineSource else { return } - - // Try to load the filter list folder. We always have to compile this at start - guard let folderURL = await setting.folderURL, - FileManager.default.fileExists(atPath: folderURL.path) - else { - return - } - - await self.compileFilterListEngineIfNeeded( - source: source, - folderURL: folderURL, - isAlwaysAggressive: setting.isAlwaysAggressive, - resourcesInfo: resourcesInfo, - compileContentBlockers: false - ) - - // Sleep for 1ms. This drastically reduces memory usage without much impact to usability - try await Task.sleep(nanoseconds: 1_000_000) - } - } catch { - // Ignore the cancellation. - } + await AdBlockGroupsManager.shared.loadFromCache() } /// Start the adblock service to get updates to the `shieldsInstallPath` @@ -110,7 +58,7 @@ public actor FilterListResourceDownloader { return } - await didUpdateResourcesComponent(folderURL: folderURL) + await AdBlockGroupsManager.shared.didUpdateResourcesComponent(folderURL: folderURL) await FilterListCustomURLDownloader.shared.startIfNeeded() if !FilterListStorage.shared.filterLists.isEmpty { @@ -122,9 +70,9 @@ public actor FilterListResourceDownloader { Task { @MainActor in for await filterListEntries in adBlockService.filterListCatalogComponentStream() { FilterListStorage.shared.loadFilterLists(from: filterListEntries) - ContentBlockerManager.log.debug("Loaded filter list catalog") - if await AdBlockStats.shared.resourcesInfo != nil { + + if AdBlockGroupsManager.shared.resourcesInfo != nil { await registerAllFilterListsIfNeeded(with: adBlockService) } } @@ -141,80 +89,31 @@ public actor FilterListResourceDownloader { } } - @discardableResult - /// When the - private func didUpdateResourcesComponent( - folderURL: URL - ) async -> CachedAdBlockEngine.ResourcesInfo { - await Task { @MainActor in - let folderSubPath = FilterListSetting.extractFolderPath(fromComponentFolderURL: folderURL) - Preferences.AppState.lastAdBlockResourcesFolderPath.value = folderSubPath - }.value - - let version = folderURL.lastPathComponent - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: folderURL.appendingPathComponent("resources.json"), - version: version - ) - - await AdBlockStats.shared.updateIfNeeded(resourcesInfo: resourcesInfo) - return resourcesInfo - } - /// Load general filter lists (shields) from the given `AdblockService` `shieldsInstallPath` `URL`. private func compileFilterListEngineIfNeeded( - source: CachedAdBlockEngine.Source, + source: GroupedAdBlockEngine.Source, folderURL: URL, - isAlwaysAggressive: Bool, - resourcesInfo: CachedAdBlockEngine.ResourcesInfo, - compileContentBlockers: Bool + engineType: GroupedAdBlockEngine.EngineType ) async { let version = folderURL.lastPathComponent - let filterListURL = folderURL.appendingPathComponent("list.txt") + let localFileURL = folderURL.appendingPathComponent("list.txt") - guard FileManager.default.fileExists(atPath: filterListURL.relativePath) else { + guard FileManager.default.fileExists(atPath: localFileURL.relativePath) else { // We are loading the old component from cache. We don't want this file to be loaded. // When we download the new component shortly we will update our cache. // This should only trigger after an app update and eventually this check can be removed. return } - let filterListInfo = CachedAdBlockEngine.FilterListInfo( + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: source, - localFileURL: filterListURL, - version: version, - fileType: .text - ) - let lazyInfo = AdBlockStats.LazyFilterListInfo( - filterListInfo: filterListInfo, - isAlwaysAggressive: isAlwaysAggressive + localFileURL: localFileURL, + version: version ) - // Check if we should load these rules - guard await AdBlockStats.shared.isEnabled(source: source) else { - await AdBlockStats.shared.updateIfNeeded( - filterListInfo: filterListInfo, - isAlwaysAggressive: isAlwaysAggressive - ) - - // To free some space, remove any rule lists that are not needed - if let blocklistType = lazyInfo.blocklistType { - do { - try await ContentBlockerManager.shared.removeRuleLists(for: blocklistType) - } catch { - ContentBlockerManager.log.error( - "Failed to remove rule lists for \(filterListInfo.debugDescription)" - ) - } - } - - return - } - - await AdBlockStats.shared.compile( - lazyInfo: lazyInfo, - resourcesInfo: resourcesInfo, - compileContentBlockers: compileContentBlockers + await AdBlockGroupsManager.shared.updated( + filterListInfo: filterListInfo, + engineType: engineType ) } @@ -227,19 +126,13 @@ public actor FilterListResourceDownloader { adBlockServiceTasks[filterList.entry.componentId] = Task { @MainActor in for await folderURL in adBlockService.register(filterList: filterList) { guard let folderURL = folderURL else { continue } - guard let resourcesInfo = await AdBlockStats.shared.resourcesInfo else { - assertionFailure("We shouldn't have started downloads before getting this value") - return - } let source = filterList.engineSource // Add or remove the filter list from the engine depending if it's been enabled or not await self.compileFilterListEngineIfNeeded( source: source, folderURL: folderURL, - isAlwaysAggressive: filterList.isAlwaysAggressive, - resourcesInfo: resourcesInfo, - compileContentBlockers: true + engineType: filterList.engineType ) // Save the downloaded folder for later (caching) purposes diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift index dca5cf0d70d9..1cd3a018b43d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift @@ -154,7 +154,7 @@ import Preferences componentId: filterList.entry.componentId, allowCreation: true, order: filterList.order, - isAlwaysAggressive: filterList.isAlwaysAggressive, + isAlwaysAggressive: filterList.engineType.isAlwaysAggressive, isDefaultEnabled: filterList.entry.defaultEnabled ) } @@ -295,7 +295,7 @@ extension AdblockFilterListCatalogEntry { extension FilterListStorage { /// Gives us source representations of all the enabled filter lists - @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + @MainActor var enabledSources: [GroupedAdBlockEngine.Source] { return filterLists.isEmpty ? allFilterListSettings .filter(\.isEnabled) @@ -304,4 +304,24 @@ extension FilterListStorage { .filter(\.isEnabled) .map(\.engineSource) } + + /// Gives us source representations of all the enabled filter lists + @MainActor func enabledSources( + engineType: GroupedAdBlockEngine.EngineType + ) -> [GroupedAdBlockEngine.Source] { + if !filterLists.isEmpty { + return filterLists.compactMap { filterList -> GroupedAdBlockEngine.Source? in + guard filterList.engineType == engineType else { return nil } + guard filterList.isEnabled else { return nil } + return filterList.engineSource + } + } else { + // We may not have the filter lists loaded yet. In which case we load the settings + return allFilterListSettings.compactMap { setting -> GroupedAdBlockEngine.Source? in + guard setting.isAlwaysAggressive == engineType.isAlwaysAggressive else { return nil } + guard setting.isEnabled else { return nil } + return setting.engineSource + } + } + } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift b/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift index 6c17f0080705..36159c14d91f 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift @@ -13,7 +13,7 @@ extension AdblockEngine { case couldNotDeserializeDATFile } - public func useResources(fromFileURL fileURL: URL) throws { + func useResources(fromFileURL fileURL: URL) throws { // Add scriplets if available if let json = try Self.validateJSON(Data(contentsOf: fileURL)) { useResources(json) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift b/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift deleted file mode 100644 index 8ef7dc579624..000000000000 --- a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift +++ /dev/null @@ -1,355 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import BraveCore -import BraveShared -import Combine -import Data -import Foundation -import Shared -import os.log - -/// This object holds on to our adblock engines and returns information needed for stats tracking as well as some conveniences -/// for injected scripts needed during web navigation and cosmetic filters models needed by the `SelectorsPollerScript.js` script. -public actor AdBlockStats { - typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) - public static let shared = AdBlockStats() - - /// An object containing the basic information to allow us to compile an engine - public struct LazyFilterListInfo { - let filterListInfo: CachedAdBlockEngine.FilterListInfo - let isAlwaysAggressive: Bool - } - - /// A list of filter list info that are available for compilation. This information is used for lazy loading. - private(set) var availableFilterLists: [CachedAdBlockEngine.Source: LazyFilterListInfo] - /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. - public private(set) var resourcesInfo: CachedAdBlockEngine.ResourcesInfo? - /// Adblock engine for general adblock lists. - private(set) var cachedEngines: [CachedAdBlockEngine.Source: CachedAdBlockEngine] - /// The current task that is compiling. - private var currentCompileTask: Task<(), Never>? - - /// Return an array of all sources that are enabled according to user's settings - /// - Note: This does not take into account the domain or global adblock toggle - @MainActor var enabledSources: [CachedAdBlockEngine.Source] { - var enabledSources = FilterListStorage.shared.enabledSources - enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) - return enabledSources - } - - init() { - cachedEngines = [:] - availableFilterLists = [:] - } - - /// Handle memory warnings by freeing up some memory - func didReceiveMemoryWarning() async { - await cachedEngines.values.asyncForEach({ await $0.clearCaches() }) - await removeDisabledEngines() - } - - /// Create and add an engine from the given resources. - /// If an engine already exists for the given source, it will be replaced. - public func compile( - lazyInfo: LazyFilterListInfo, - resourcesInfo: CachedAdBlockEngine.ResourcesInfo, - compileContentBlockers: Bool - ) async { - await currentCompileTask?.value - - currentCompileTask = Task.detached { - // Compile engine - if await self.needsCompilation(for: lazyInfo.filterListInfo, resourcesInfo: resourcesInfo) { - do { - let engine = try CachedAdBlockEngine.compile( - filterListInfo: lazyInfo.filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: lazyInfo.isAlwaysAggressive - ) - - await self.add(engine: engine) - } catch { - ContentBlockerManager.log.error( - "Failed to compile engine for \(lazyInfo.filterListInfo.source.debugDescription)" - ) - } - } - - // Compile content blockers - if compileContentBlockers, let blocklistType = lazyInfo.blocklistType { - let modes = await ContentBlockerManager.shared.missingModes(for: blocklistType) - guard !modes.isEmpty else { return } - - do { - try await ContentBlockerManager.shared.compileRuleList( - at: lazyInfo.filterListInfo.localFileURL, - for: blocklistType, - modes: modes - ) - } catch { - ContentBlockerManager.log.error( - "Failed to compile rule list for \(lazyInfo.filterListInfo.source.debugDescription)" - ) - } - } - } - - await currentCompileTask?.value - } - - /// Add a new engine to the list. - /// If an engine already exists for the same source, it will be replaced instead. - private func add(engine: CachedAdBlockEngine) { - cachedEngines[engine.filterListInfo.source] = engine - updateIfNeeded(resourcesInfo: engine.resourcesInfo) - updateIfNeeded( - filterListInfo: engine.filterListInfo, - isAlwaysAggressive: engine.isAlwaysAggressive - ) - let typeString = engine.isAlwaysAggressive ? "aggressive" : "standard" - ContentBlockerManager.log.debug( - "Added \(typeString) engine for \(engine.filterListInfo.debugDescription)" - ) - } - - /// Add or update `filterListInfo` if it is a newer version. This information is used for lazy loading. - func updateIfNeeded(filterListInfo: CachedAdBlockEngine.FilterListInfo, isAlwaysAggressive: Bool) - { - if let existingLazyInfo = availableFilterLists[filterListInfo.source] { - guard filterListInfo.version > existingLazyInfo.filterListInfo.version else { return } - } - - availableFilterLists[filterListInfo.source] = LazyFilterListInfo( - filterListInfo: filterListInfo, - isAlwaysAggressive: isAlwaysAggressive - ) - } - - /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. - func updateIfNeeded(resourcesInfo: CachedAdBlockEngine.ResourcesInfo) { - guard self.resourcesInfo == nil || resourcesInfo.version > self.resourcesInfo!.version else { - return - } - self.resourcesInfo = resourcesInfo - - if #available(iOS 16.0, *) { - ContentBlockerManager.log.debug( - "Updated resources component: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" - ) - } else { - ContentBlockerManager.log.debug( - "Updated resources component: `\(resourcesInfo.localFileURL.path)`" - ) - } - } - - /// Remove all the engines - func removeAllEngines() { - cachedEngines.removeAll() - } - - /// Remove all engines that have disabled sources - func removeDisabledEngines() async { - let sources = await Set(enabledSources) - - for source in cachedEngines.keys { - guard !sources.contains(source) else { continue } - // Remove the engine - if let filterListInfo = cachedEngines[source]?.filterListInfo { - cachedEngines.removeValue(forKey: source) - ContentBlockerManager.log.debug("Removed engine for \(filterListInfo.debugDescription)") - } - - // Delete the Content blockers - if let lazyInfo = availableFilterLists[source], let blocklistType = lazyInfo.blocklistType { - do { - try await ContentBlockerManager.shared.removeRuleLists(for: blocklistType) - } catch { - ContentBlockerManager.log.error( - "Failed to remove rule lists for \(lazyInfo.filterListInfo.debugDescription)" - ) - } - } - } - } - - /// Remove all engines that have disabled sources - func ensureEnabledEngines() async { - do { - for source in await enabledSources { - guard cachedEngines[source] == nil else { continue } - guard let availableFilterList = availableFilterLists[source] else { continue } - guard let resourcesInfo = self.resourcesInfo else { continue } - - await compile( - lazyInfo: availableFilterList, - resourcesInfo: resourcesInfo, - compileContentBlockers: true - ) - - // Sleep for 1ms. This drastically reduces memory usage without much impact to usability - try await Task.sleep(nanoseconds: 1_000_000) - } - } catch { - // Ignore cancellation errors - } - } - - /// Tells us if this source should be loaded. - @MainActor func isEnabled(source: CachedAdBlockEngine.Source) -> Bool { - return enabledSources.contains(source) - } - - /// Tells us if an engine needs compilation if it's missing or if its resources are outdated - func needsCompilation( - for filterListInfo: CachedAdBlockEngine.FilterListInfo, - resourcesInfo: CachedAdBlockEngine.ResourcesInfo - ) -> Bool { - if let cachedEngine = cachedEngines[filterListInfo.source] { - return cachedEngine.filterListInfo.version < filterListInfo.version - && cachedEngine.resourcesInfo.version < resourcesInfo.version - } else { - return true - } - } - - /// Checks the general and regional engines to see if the request should be blocked - func shouldBlock( - requestURL: URL, - sourceURL: URL, - resourceType: AdblockEngine.ResourceType, - isAggressiveMode: Bool - ) async -> Bool { - let sources = await self.enabledSources - return await cachedEngines(for: sources).asyncConcurrentMap({ cachedEngine in - return await cachedEngine.shouldBlock( - requestURL: requestURL, - sourceURL: sourceURL, - resourceType: resourceType, - isAggressiveMode: isAggressiveMode - ) - }).contains(where: { $0 }) - } - - /// This returns all the user script types for the given frame - func makeEngineScriptTypes( - frameURL: URL, - isMainFrame: Bool, - isDeAmpEnabled: Bool, - domain: Domain - ) async -> Set { - // Add any engine scripts for this frame - return await cachedEngines(for: domain).enumerated().asyncMap({ - index, - cachedEngine -> Set in - do { - return try await cachedEngine.makeEngineScriptTypes( - frameURL: frameURL, - isMainFrame: isMainFrame, - domain: domain, - isDeAmpEnabled: isDeAmpEnabled, - index: index - ) - } catch { - assertionFailure() - return [] - } - }).reduce( - Set(), - { partialResult, scriptTypes in - return partialResult.union(scriptTypes) - } - ) - } - - /// Returns all appropriate engines for the given domain - @MainActor func cachedEngines(for domain: Domain) async -> [CachedAdBlockEngine] { - let sources = enabledSources(for: domain) - return await cachedEngines(for: sources) - } - - /// Return all the cached engines for the given sources. If any filter list is not yet loaded, it will be lazily loaded - private func cachedEngines(for sources: [CachedAdBlockEngine.Source]) -> [CachedAdBlockEngine] { - return sources.compactMap { source -> CachedAdBlockEngine? in - return cachedEngines[source] - } - } - - /// Returns all the models for this frame URL - func cosmeticFilterModels( - forFrameURL frameURL: URL, - domain: Domain - ) async -> [CosmeticFilterModelTuple] { - return await cachedEngines(for: domain).asyncConcurrentCompactMap { - cachedEngine -> CosmeticFilterModelTuple? in - do { - guard let model = try await cachedEngine.cosmeticFilterModel(forFrameURL: frameURL) else { - return nil - } - return (cachedEngine.isAlwaysAggressive, model) - } catch { - assertionFailure() - return nil - } - } - } - - /// Give us all the enabled sources for the given domain - @MainActor private func enabledSources(for domain: Domain) -> [CachedAdBlockEngine.Source] { - let enabledSources = self.enabledSources - return enabledSources.filter({ $0.isEnabled(for: domain) }) - } -} - -extension FilterListSetting { - @MainActor var engineSource: CachedAdBlockEngine.Source? { - guard let componentId = componentId else { return nil } - return .filterList(componentId: componentId, uuid: uuid) - } -} - -extension FilterList { - @MainActor var engineSource: CachedAdBlockEngine.Source { - return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) - } -} - -extension CustomFilterListSetting { - @MainActor var engineSource: CachedAdBlockEngine.Source { - return .filterListURL(uuid: uuid) - } -} - -extension CachedAdBlockEngine.Source { - /// Returns a boolean indicating if the engine is enabled for the given domain. - /// - /// This is determined by checking the source of the engine and checking the appropriate shields. - @MainActor fileprivate func isEnabled(for domain: Domain) -> Bool { - switch self { - case .filterList, .filterListURL: - // This engine source type is enabled only if shields are enabled - // for the given domain - return domain.isShieldExpected(.adblockAndTp, considerAllShieldsOption: true) - } - } -} - -extension AdBlockStats.LazyFilterListInfo { - var blocklistType: ContentBlockerManager.BlocklistType? { - switch filterListInfo.source { - case .filterList(let componentId, let uuid): - guard uuid != FilterList.defaultComponentUUID else { - // For now we don't compile this into content blockers because we use the one coming from slim list - // We might change this in the future as it ends up with 95k items whereas the limit is 150k. - // So there is really no reason to use slim list except perhaps for performance which we need to test out. - return nil - } - - return .filterList(componentId: componentId, isAlwaysAggressive: isAlwaysAggressive) - case .filterListURL(let uuid): - return .customFilterList(uuid: uuid) - } - } -} diff --git a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift index 0c3a40553b2c..b9af2b638290 100644 --- a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift +++ b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift @@ -16,7 +16,19 @@ final class PageDataTests: XCTestCase { let mainFrameURL = URL(string: "http://example.com")! let subFrameURL = URL(string: "http://example.com/1p/subframe")! let upgradedMainFrameURL = URL(string: "https://example.com")! - var pageData = PageData(mainFrameURL: mainFrameURL, adBlockStats: AdBlockStats()) + var pageData = PageData( + mainFrameURL: mainFrameURL, + groupsManager: AdBlockGroupsManager( + standardManager: AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ), + aggressiveManager: AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_aggressive" + ) + ) + ) let expectation = expectation(description: "") Task { @MainActor in diff --git a/ios/brave-ios/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift similarity index 69% rename from ios/brave-ios/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift rename to ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift index 0592bc26a78c..5046e5b0f42d 100644 --- a/ios/brave-ios/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift +++ b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift @@ -9,7 +9,7 @@ import XCTest @testable import Brave -final class CachedAdBlockEngineTests: XCTestCase { +final class GroupedAdBlockEngineTests: XCTestCase { func test3rdPartyCheck() throws { let engine = try AdblockEngine( rules: [ @@ -61,27 +61,22 @@ final class CachedAdBlockEngineTests: XCTestCase { AdblockEngine.setDomainResolver() var engine: AdblockEngine? = AdblockEngine() weak var weakEngine: AdblockEngine? = engine + let localFileURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt" + )! - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, - version: "bundled" - ) - - let filterListInfo = CachedAdBlockEngine.FilterListInfo( + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), - localFileURL: Bundle.module.url( - forResource: "iodkpdagapdfkphljnddpjlldadblomo", - withExtension: "txt" - )!, - version: "bundled", - fileType: .text + localFileURL: localFileURL, + version: "bundled" ) - - var cachedEngine: CachedAdBlockEngine? = CachedAdBlockEngine( + + var cachedEngine: GroupedAdBlockEngine? = GroupedAdBlockEngine( engine: engine!, - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: false + group: GroupedAdBlockEngine.FilterListGroup( + infos: [filterListInfo], localFileURL: localFileURL, fileType: .text + ), + type: .standard ) XCTAssertNotNil(cachedEngine) @@ -96,19 +91,20 @@ final class CachedAdBlockEngineTests: XCTestCase { } func testCompilationofResources() throws { - let textFilterListInfo = CachedAdBlockEngine.FilterListInfo( + let localFileURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt" + )! + let textFilterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), - localFileURL: Bundle.module.url( - forResource: "iodkpdagapdfkphljnddpjlldadblomo", - withExtension: "txt" - )!, - version: "bundled", - fileType: .text - ) - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + localFileURL: localFileURL, version: "bundled" ) + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, version: "bundled" + ) + let group = GroupedAdBlockEngine.FilterListGroup( + infos: [textFilterListInfo], localFileURL: localFileURL, fileType: .text + ) let expectation = expectation(description: "Compiled engine resources") AdblockEngine.setDomainResolver() @@ -120,12 +116,8 @@ final class CachedAdBlockEngineTests: XCTestCase { await filterListInfos.asyncConcurrentForEach { filterListInfo in do { - let engine = try CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: false - ) - + let engine = try GroupedAdBlockEngine.compile(group: group, type: .standard) + try await engine.useResources(from: resourcesInfo) let url = URL(string: "https://stackoverflow.com")! let domain = await MainActor.run { @@ -133,22 +125,16 @@ final class CachedAdBlockEngineTests: XCTestCase { } let sameDomainTypes = try await engine.makeEngineScriptTypes( - frameURL: url, - isMainFrame: true, - domain: domain, - isDeAmpEnabled: false, - index: 0 + frameURL: url, isMainFrame: true, domain: domain, isDeAmpEnabled: false, index: 0 ) // We should have no scripts injected XCTAssertEqual(sameDomainTypes.count, 0) - if await engine.filterListInfo == textFilterListInfo { + if await engine.group.infos.contains(textFilterListInfo) { // This engine file contains some scriplet rules so we can test this part is working let crossDomainTypes = try await engine.makeEngineScriptTypes( - frameURL: URL(string: "https://reddit.com")!, - isMainFrame: true, - domain: domain, + frameURL: URL(string: "https://reddit.com")!, isMainFrame: true, domain: domain, isDeAmpEnabled: false, index: 0 ) @@ -177,15 +163,7 @@ final class CachedAdBlockEngineTests: XCTestCase { func testPerformance() throws { // Given // Ad block data and an engine manager - let sampleFilterListURL = Bundle.module.url( - forResource: "iodkpdagapdfkphljnddpjlldadblomo", - withExtension: "txt" - )! - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, - version: "bundled" - ) - + let sampleFilterListURL = Bundle.module.url(forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt")! // Then // Measure performance let options = XCTMeasureOptions() @@ -194,19 +172,18 @@ final class CachedAdBlockEngineTests: XCTestCase { measure(metrics: [XCTClockMetric(), XCTCPUMetric(), XCTMemoryMetric()], options: options) { let uuid = UUID().uuidString - let filterListInfo = CachedAdBlockEngine.FilterListInfo( + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterListURL(uuid: uuid), localFileURL: sampleFilterListURL, - version: "bundled", - fileType: .text + version: "bundled" ) - + + let group = GroupedAdBlockEngine.FilterListGroup( + infos: [filterListInfo], localFileURL: sampleFilterListURL, fileType: .text + ) + do { - _ = try CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: false - ) + _ = try GroupedAdBlockEngine.compile(group: group, type: .standard) } catch { XCTFail(error.localizedDescription) } From 6ae29988d5b89a1fcf10b7dac81f44c54b9d25e8 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Wed, 6 Mar 2024 22:32:57 +0100 Subject: [PATCH 02/10] Load dat files from cache --- .../Browser/Helpers/LaunchHelper.swift | 12 +- .../AdBlock/AdBlockEngineManager.swift | 195 +++++++++++------- .../AdBlock/AdBlockGroupsManager.swift | 48 ++--- .../AdBlock/GroupedAdBlockEngine.swift | 6 +- .../FilterListCustomURLDownloader.swift | 9 +- .../FilterListResourceDownloader.swift | 19 +- .../GroupedAdBlockEngineTests.swift | 5 +- 7 files changed, 158 insertions(+), 136 deletions(-) diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift index f4e11c92f05a..045efb7a1d18 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift @@ -48,11 +48,16 @@ public actor LaunchHelper { // Load cached data // This is done first because compileResources need their results - async let filterListCache: Void = FilterListResourceDownloader.shared - .loadFilterListSettingsAndCachedData() + await FilterListStorage.shared.loadFilterListSettings() + await AdBlockGroupsManager.shared.loadResourcesFromCache() + + // Only load the standard engine for now, we will load the other one after launch + async let loadEngineFromCache: Void = AdBlockGroupsManager.shared.loadEngineFromCache( + for: .standard + ) async let adblockResourceCache: Void = AdblockResourceDownloader.shared .loadCachedAndBundledDataIfNeeded(allowedModes: launchBlockModes) - _ = await (filterListCache, adblockResourceCache) + _ = await (loadEngineFromCache, adblockResourceCache) Self.signpost.emitEvent("loadedCachedData", id: signpostID, "Loaded cached data") ContentBlockerManager.log.debug("Loaded blocking launch data") @@ -114,6 +119,7 @@ public actor LaunchHelper { id: signpostID, "Reloaded data for remaining modes" ) + await AdBlockGroupsManager.shared.loadEngineFromCache(for: .aggressive) await AdblockResourceDownloader.shared.startFetching() Self.signpost.emitEvent("startFetching", id: signpostID, "Started fetching ad-block data") diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index 7ec5c5a6a37f..6e4c6d9894f9 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -16,10 +16,15 @@ import os return FileManager.SearchPathDirectory.applicationSupportDirectory } + public struct FileInfo: Hashable, Equatable { + let filterListInfo: GroupedAdBlockEngine.FilterListInfo + let localFileURL: URL + } + /// The top level folder to create for caching engine data private static let parentCacheFolderName = "engines" /// All the info that is currently available - private var availableInfos: [GroupedAdBlockEngine.FilterListInfo] + private var availableFiles: [FileInfo] /// The subfolder that the cache data is stored. /// Since we can have multiple engines this folder will be unique per engine private var cacheFolderName: String @@ -52,23 +57,6 @@ import os } } - /// Get the url of the cached file - /// - /// - Note: Returns nil if the file does not exist - private var savedCombinedListURL: URL? { - guard let cacheFolderURL = createdCacheFolderURL else { - return nil - } - - let fileURL = cacheFolderURL.appendingPathComponent("list.txt", conformingTo: .text) - - if FileManager.default.fileExists(atPath: fileURL.path) { - return fileURL - } else { - return nil - } - } - /// Return an array of all sources that are enabled according to user's settings /// - Note: This does not take into account the domain or global adblock toggle private var enabledSources: [GroupedAdBlockEngine.Source] { @@ -80,14 +68,14 @@ import os } /// All the infos that are compilable based on the enabled sources and available infos - var compilableInfos: [GroupedAdBlockEngine.FilterListInfo] { + var compilableFiles: [FileInfo] { return enabledSources.compactMap { source in - return availableInfos.first(where: { $0.source == source }) + return availableFiles.first(where: { $0.filterListInfo.source == source }) } } init(engineType: GroupedAdBlockEngine.EngineType, cacheFolderName: String) { - self.availableInfos = [] + self.availableFiles = [] self.engine = nil self.engineType = engineType self.cacheFolderName = cacheFolderName @@ -99,48 +87,59 @@ import os } /// Add the info to the available list - func add(info: GroupedAdBlockEngine.FilterListInfo) { - availableInfos.removeAll { exisitingInfo in - return exisitingInfo.source == info.source && exisitingInfo.version <= info.version + func add(fileInfo: FileInfo) { + availableFiles.removeAll { existingFileInfo in + return existingFileInfo.filterListInfo == fileInfo.filterListInfo } - availableInfos.append(info) + availableFiles.append(fileInfo) } /// Remove any info from the available list given by the source /// Mostly used for custom filter lists func removeInfo(for source: GroupedAdBlockEngine.Source) { - availableInfos.removeAll { exisitingInfo in - return exisitingInfo.source == source + availableFiles.removeAll { fileInfo in + return fileInfo.filterListInfo.source == source } } /// Checks to see if we need to compile or recompile the engine based on the available info func checkNeedsCompile() -> Bool { - let compilableInfos = compilableInfos + guard let engine = engine else { return true } + let compilableInfos = compilableFiles.map({ $0.filterListInfo }) guard !compilableInfos.isEmpty else { return false } - return compilableInfos != engine?.group.infos + return compilableInfos != engine.group.infos + } + + /// Checks to see if we need to compile or recompile the engine based on the available info + func checkHasAllInfo() -> Bool { + return compilableFiles == availableFiles } /// Load the engine from cache so it can be ready during launch - func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async throws { - guard let localFileURL = savedCombinedListURL else { return } - guard let cacheInfo = loadCachedInfo() else { return } - - let groupedEngine = try GroupedAdBlockEngine.compile( - group: GroupedAdBlockEngine.FilterListGroup( - infos: cacheInfo.infos, - localFileURL: localFileURL, - fileType: cacheInfo.fileType - ), - type: engineType - ) + func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async { + do { + guard let cachedGroupInfo = loadCachedInfo() else { return } + let engineType = self.engineType + let groupedEngine = try await Task.detached(priority: .high) { + let engine = try GroupedAdBlockEngine.compile( + group: cachedGroupInfo, + type: engineType + ) - if let resourcesInfo = resourcesInfo { - try await groupedEngine.useResources(from: resourcesInfo) - } + if let resourcesInfo = resourcesInfo { + try await engine.useResources(from: resourcesInfo) + } - self.set(engine: groupedEngine) + return engine + }.value + + self.set(engine: groupedEngine) + } catch { + ContentBlockerManager.log.error( + "Failed to load engine from cache for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + } } /// This is a task that we use to delay a requested compile. @@ -152,13 +151,15 @@ import os /// This will compile available data, but will wait a little bit in case something new gets downloaded. /// Especially needed during launch when we have a bunch of downloads coming at the same time. - func compileDelayed(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) { + func compileDelayedIfNeeded(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) { // Cancel the previous task delayTask?.cancel() // Restart the task delayTask = Task { - try await Task.sleep(seconds: 10) + let hasAllInfo = checkHasAllInfo() + try await Task.sleep(seconds: hasAllInfo ? 10 : 60) + guard checkNeedsCompile() else { return } try await compileAvailable(resourcesInfo: resourcesInfo) } } @@ -167,38 +168,50 @@ import os func compileAvailable(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async throws { // 1. Create the group let group = try combineRules() + let engineType = self.engineType self.pendingGroup = group // 2. Compile the engine - let groupedEngine = try GroupedAdBlockEngine.compile(group: group, type: engineType) - if let resourcesInfo { - try await groupedEngine.useResources(from: resourcesInfo) - } + let groupedEngine = try await Task.detached(priority: .background) { + let engine = try GroupedAdBlockEngine.compile(group: group, type: engineType) + + if let resourcesInfo { + try await engine.useResources(from: resourcesInfo) + } + + return engine + }.value // 3. Ensure our file is still up to date before setting it // (avoid race conditiions) guard pendingGroup == group else { return } self.set(engine: groupedEngine) - saveCachedInfo(from: groupedEngine) + await cache(engine: groupedEngine) self.pendingGroup = nil } private func set(engine: GroupedAdBlockEngine) { let infosString = engine.group.infos.map({ " \($0.debugDescription)" }).joined(separator: "\n") ContentBlockerManager.log.debug( - "Set `\(self.cacheFolderName)` engine from \(engine.group.infos.count) sources:\n\(infosString)" + "Set `\(self.cacheFolderName)` (\(engine.group.fileType.debugDescription)) engine from \(engine.group.infos.count) sources:\n\(infosString)" ) self.engine = engine } /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. - func update(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) async throws { - try await engine?.useResources(from: resourcesInfo) + func update(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) async { + do { + try await engine?.useResources(from: resourcesInfo) + } catch { + ContentBlockerManager.log.error( + "Failed to update `\(self.cacheFolderName)` engne with resources" + ) + } } - func ensureContentBlockers(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) async { + func ensureContentBlockers(for fileInfo: FileInfo) async { guard - let blocklistType = filterListInfo.source.blocklistType( + let blocklistType = fileInfo.filterListInfo.source.blocklistType( isAlwaysAggressive: engineType.isAlwaysAggressive ) else { @@ -210,13 +223,13 @@ import os do { try await ContentBlockerManager.shared.compileRuleList( - at: filterListInfo.localFileURL, + at: fileInfo.localFileURL, for: blocklistType, modes: modes ) } catch { ContentBlockerManager.log.error( - "Failed to compile rule list for \(filterListInfo.debugDescription)" + "Failed to compile rule list for \(fileInfo.filterListInfo.debugDescription)" ) } } @@ -224,23 +237,25 @@ import os /// Take all the filter lists and combine them into one then save them into a cache folder. private func combineRules() throws -> GroupedAdBlockEngine.FilterListGroup { // 1. Grab all the needed infos - let infos = compilableInfos + let compilableFiles = compilableFiles // 2. Create a file url let cachedFolder = try getOrCreateCacheFolder() let fileURL = cachedFolder.appendingPathComponent("list.txt", conformingTo: .text) - + var compiledInfos: [GroupedAdBlockEngine.FilterListInfo] = [] + var unifiedRules = "" // 3. Join all the rules together - let unifiedRules = infos.compactMap { info in + compilableFiles.forEach { fileInfo in do { - return try String(contentsOf: info.localFileURL) + let fileContents = try String(contentsOf: fileInfo.localFileURL) + compiledInfos.append(fileInfo.filterListInfo) + unifiedRules = [unifiedRules, fileContents].joined(separator: "\n") } catch { ContentBlockerManager.log.error( - "Could not load rules for `\(info.debugDescription)`: \(error)" + "Could not load rules for `\(fileInfo.filterListInfo.debugDescription)`: \(error)" ) - return nil } - }.joined(separator: "\n") + } // 4. Save the files into storage if FileManager.default.fileExists(atPath: fileURL.path) { @@ -250,7 +265,7 @@ import os // 4. Return a group containing info on this new file return GroupedAdBlockEngine.FilterListGroup( - infos: infos, + infos: compiledInfos, localFileURL: fileURL, fileType: .text ) @@ -272,20 +287,32 @@ import os return folderURL } - private func saveCachedInfo(from engine: GroupedAdBlockEngine) { + private func cache(engine: GroupedAdBlockEngine) async { let encoder = JSONEncoder() - let info = CachedEngineInfo(infos: engine.group.infos, fileType: engine.group.fileType) do { - let data = try encoder.encode(info) let folderURL = try getOrCreateCacheFolder() - let fileURL = folderURL.appendingPathComponent("engine_info", conformingTo: .json) - if FileManager.default.fileExists(atPath: fileURL.path) { - try FileManager.default.removeItem(at: fileURL) + // Write the serialized engine + let serializedEngine = try await engine.serialize() + let serializedEngineURL = folderURL.appendingPathComponent("list.dat", conformingTo: .data) + + if FileManager.default.fileExists(atPath: serializedEngineURL.path) { + try FileManager.default.removeItem(at: serializedEngineURL) + } + + try serializedEngine.write(to: serializedEngineURL) + + // Write the info about the engine + let info = CachedEngineInfo(infos: engine.group.infos, fileType: .data) + let data = try encoder.encode(info) + let infoFileURL = folderURL.appendingPathComponent("engine_info.json", conformingTo: .json) + + if FileManager.default.fileExists(atPath: infoFileURL.path) { + try FileManager.default.removeItem(at: infoFileURL) } - try data.write(to: fileURL) + try data.write(to: infoFileURL) } catch { ContentBlockerManager.log.error( "Failed to save cache info for `\(self.cacheFolderName)`: \(String(describing: error))" @@ -293,15 +320,23 @@ import os } } - private func loadCachedInfo() -> CachedEngineInfo? { + private func loadCachedInfo() -> GroupedAdBlockEngine.FilterListGroup? { + guard let cacheFolderURL = createdCacheFolderURL else { return nil } + let cachedEngineURL = cacheFolderURL.appendingPathComponent("list.dat", conformingTo: .data) + guard FileManager.default.fileExists(atPath: cachedEngineURL.path) else { return nil } + let cachedInfoURL = cacheFolderURL.appendingPathComponent("engine_info", conformingTo: .json) + guard FileManager.default.fileExists(atPath: cachedInfoURL.path) else { return nil } let decoder = JSONDecoder() do { - let folderURL = try getOrCreateCacheFolder() - let fileURL = folderURL.appendingPathComponent("engine_info", conformingTo: .json) - guard FileManager.default.fileExists(atPath: fileURL.path) else { return nil } - let data = try Data(contentsOf: fileURL) - return try decoder.decode(CachedEngineInfo.self, from: data) + let data = try Data(contentsOf: cachedInfoURL) + let cachedInfo = try decoder.decode(CachedEngineInfo.self, from: data) + + return GroupedAdBlockEngine.FilterListGroup( + infos: cachedInfo.infos, + localFileURL: cachedEngineURL, + fileType: cachedInfo.fileType + ) } catch { ContentBlockerManager.log.error( "Failed to load cache info for `\(self.cacheFolderName)`: \(String(describing: error))" diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 933118ec9804..5291fc035a48 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -41,7 +41,7 @@ import os } /// Load any cache data so its ready right during launch - func loadFromCache() async { + func loadResourcesFromCache() async { if let resourcesFolderURL = FilterListSetting.makeFolderURL( forComponentFolderPath: Preferences.AppState.lastAdBlockResourcesFolderPath.value ), FileManager.default.fileExists(atPath: resourcesFolderURL.path), @@ -49,19 +49,20 @@ import os { // We need this for all filter lists so we can't compile anything until we download it self.resourcesInfo = resourcesInfo - } - - await allManagers.asyncConcurrentForEach { manager in - guard manager.engineType.loadFromCache else { return } - - do { - try await manager.loadFromCache(resourcesInfo: self.resourcesInfo) - } catch { - ContentBlockerManager.log.error("Failed to load engine from cache: \(error)") + + if #available(iOS 16.0, *) { + ContentBlockerManager.log.debug( + "Loaded resources component from cache: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" + ) } } } + func loadEngineFromCache(for engineType: GroupedAdBlockEngine.EngineType) async { + let manager = getManager(for: engineType) + await manager.loadFromCache(resourcesInfo: resourcesInfo) + } + /// Inform this manager of updates to the resources so our engines can be updated func didUpdateResourcesComponent(folderURL: URL) async { await Task { @MainActor in @@ -80,20 +81,17 @@ import os /// Handle updated filter list info func updated( - filterListInfo: GroupedAdBlockEngine.FilterListInfo, + fileInfo: AdBlockEngineManager.FileInfo, engineType: GroupedAdBlockEngine.EngineType ) async { let manager = getManager(for: engineType) // Always update the info on the manager - manager.add(info: filterListInfo) - - if manager.checkNeedsCompile() { - manager.compileDelayed(resourcesInfo: resourcesInfo) - } + manager.add(fileInfo: fileInfo) + manager.compileDelayedIfNeeded(resourcesInfo: resourcesInfo) // Compile content blockers if this filter list is enabled - if manager.isEnabled(source: filterListInfo.source) { - await manager.ensureContentBlockers(for: filterListInfo) + if manager.isEnabled(source: fileInfo.filterListInfo.source) { + await manager.ensureContentBlockers(for: fileInfo) } } @@ -112,8 +110,9 @@ import os } // Compile all content blockers for the given manager - await manager.compilableInfos.asyncForEach { filterListInfo in - await manager.ensureContentBlockers(for: filterListInfo) + await manager.compilableFiles.asyncForEach { fileInfo in + guard manager.isEnabled(source: fileInfo.filterListInfo.source) else { return } + await manager.ensureContentBlockers(for: fileInfo) } } } @@ -136,7 +135,7 @@ import os allManagers.forEach { manager in Task { - try await manager.update(resourcesInfo: resourcesInfo) + await manager.update(resourcesInfo: resourcesInfo) } } @@ -236,13 +235,6 @@ extension GroupedAdBlockEngine.EngineType { } } - fileprivate var loadFromCache: Bool { - switch self { - case .standard: return true - case .aggressive: return false - } - } - @MainActor fileprivate func makeDefaultManager() -> AdBlockEngineManager { return AdBlockEngineManager(engineType: self, cacheFolderName: defaultCachedFolderName) } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift index 9d4da7edd68a..ae0c601eb489 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift @@ -49,7 +49,6 @@ public actor GroupedAdBlockEngine { public struct FilterListInfo: Codable, Hashable, Equatable, CustomDebugStringConvertible { let source: GroupedAdBlockEngine.Source - let localFileURL: URL let version: String public var debugDescription: String { @@ -195,6 +194,11 @@ public actor GroupedAdBlockEngine { resourcesInfo = info } + /// Serialize the engine into data to be later loaded from cache + public func serialize() throws -> Data { + return try engine.serialize() + } + /// Create an engine from the given resources public static func compile( group: FilterListGroup, diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift index b31b7e0698aa..c9cd286fafdd 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift @@ -71,14 +71,13 @@ actor FilterListCustomURLDownloader: ObservableObject { let source = await filterListCustomURL.setting.engineSource let version = fileVersionDateFormatter.string(from: downloadResult.date) - let filterListInfo = GroupedAdBlockEngine.FilterListInfo( - source: source, - localFileURL: downloadResult.fileURL, - version: version + let fileInfo = AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo(source: source, version: version), + localFileURL: downloadResult.fileURL ) await AdBlockGroupsManager.shared.updated( - filterListInfo: filterListInfo, + fileInfo: fileInfo, engineType: .aggressive ) } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index 587669af33cd..d3d1856ad0e8 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -32,16 +32,6 @@ public actor FilterListResourceDownloader { self.adBlockService = nil } - /// This loads the filter list settings from core data. - /// It uses these settings and other stored properties to load the enabled general shields and filter lists. - /// - /// - Warning: This method loads filter list settings. - /// You need to wait for `DataController.shared.initializeOnce()` to be called first before invoking this method - public func loadFilterListSettingsAndCachedData() async { - await FilterListStorage.shared.loadFilterListSettings() - await AdBlockGroupsManager.shared.loadFromCache() - } - /// Start the adblock service to get updates to the `shieldsInstallPath` public func start(with adBlockService: AdblockService) { self.adBlockService = adBlockService @@ -105,14 +95,13 @@ public actor FilterListResourceDownloader { return } - let filterListInfo = GroupedAdBlockEngine.FilterListInfo( - source: source, - localFileURL: localFileURL, - version: version + let fileInfo = AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo(source: source, version: version), + localFileURL: localFileURL ) await AdBlockGroupsManager.shared.updated( - filterListInfo: filterListInfo, + fileInfo: fileInfo, engineType: engineType ) } diff --git a/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift index 5046e5b0f42d..d0203c7681b0 100644 --- a/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift +++ b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift @@ -67,7 +67,6 @@ final class GroupedAdBlockEngineTests: XCTestCase { let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), - localFileURL: localFileURL, version: "bundled" ) @@ -95,8 +94,7 @@ final class GroupedAdBlockEngineTests: XCTestCase { forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt" )! let textFilterListInfo = GroupedAdBlockEngine.FilterListInfo( - source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), - localFileURL: localFileURL, + source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), version: "bundled" ) let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( @@ -174,7 +172,6 @@ final class GroupedAdBlockEngineTests: XCTestCase { let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterListURL(uuid: uuid), - localFileURL: sampleFilterListURL, version: "bundled" ) From 00151da4232b4179c25b3f937ce3d69d428beade Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Mon, 25 Mar 2024 21:49:12 +0100 Subject: [PATCH 03/10] Load filter lists from cache using legacy storage --- .../AdBlock/AdBlockEngineManager.swift | 13 ++++-- .../AdBlock/AdBlockGroupsManager.swift | 46 ++++++++++++++++++- .../FilterListResourceDownloader.swift | 18 +++----- .../Brave/WebFilters/FilterListStorage.swift | 1 + 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index 6e4c6d9894f9..8a026693e53d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -105,9 +105,9 @@ import os /// Checks to see if we need to compile or recompile the engine based on the available info func checkNeedsCompile() -> Bool { - guard let engine = engine else { return true } let compilableInfos = compilableFiles.map({ $0.filterListInfo }) guard !compilableInfos.isEmpty else { return false } + guard let engine = engine else { return true } return compilableInfos != engine.group.infos } @@ -117,9 +117,9 @@ import os } /// Load the engine from cache so it can be ready during launch - func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async { + func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async -> Bool { do { - guard let cachedGroupInfo = loadCachedInfo() else { return } + guard let cachedGroupInfo = loadCachedInfo() else { return false } let engineType = self.engineType let groupedEngine = try await Task.detached(priority: .high) { let engine = try GroupedAdBlockEngine.compile( @@ -135,10 +135,13 @@ import os }.value self.set(engine: groupedEngine) + return true } catch { ContentBlockerManager.log.error( "Failed to load engine from cache for `\(self.cacheFolderName)`: \(String(describing: error))" ) + + return false } } @@ -192,8 +195,10 @@ import os private func set(engine: GroupedAdBlockEngine) { let infosString = engine.group.infos.map({ " \($0.debugDescription)" }).joined(separator: "\n") + let fileTypeString = engine.group.fileType.debugDescription + let count = engine.group.infos.count ContentBlockerManager.log.debug( - "Set `\(self.cacheFolderName)` (\(engine.group.fileType.debugDescription)) engine from \(engine.group.infos.count) sources:\n\(infosString)" + "Set `\(self.cacheFolderName)` (\(fileTypeString)) engine from \(count) sources:\n\(infosString)" ) self.engine = engine } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 5291fc035a48..45d5e31dde06 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -60,7 +60,51 @@ import os func loadEngineFromCache(for engineType: GroupedAdBlockEngine.EngineType) async { let manager = getManager(for: engineType) - await manager.loadFromCache(resourcesInfo: resourcesInfo) + + if await !manager.loadFromCache(resourcesInfo: self.resourcesInfo) { + // If we didn't load the main engine from cache we need to load using the old cache mechanism + // This is only temporary so we're not left with no ad-block during the upgrade. + // We can drop all of this in future upgrades as by then we will have files cached in the new format + for setting in FilterListStorage.shared.allFilterListSettings + .filter({ $0.isAlwaysAggressive == engineType.isAlwaysAggressive }) + .sorted(by: { $0.order?.intValue ?? 0 <= $1.order?.intValue ?? 0 }) + { + guard let folderURL = setting.folderURL else { continue } + guard let source = setting.engineSource else { continue } + guard let fileInfo = Self.fileInfo(for: source, folderURL: folderURL) else { continue } + manager.add(fileInfo: fileInfo) + } + + if manager.checkNeedsCompile() { + do { + try await manager.compileAvailable(resourcesInfo: self.resourcesInfo) + } catch { + + } + } + } + } + + static func fileInfo( + for source: GroupedAdBlockEngine.Source, + folderURL: URL + ) -> AdBlockEngineManager.FileInfo? { + let version = folderURL.lastPathComponent + let localFileURL = folderURL.appendingPathComponent("list.txt") + + guard FileManager.default.fileExists(atPath: localFileURL.relativePath) else { + // We are loading the old component from cache. We don't want this file to be loaded. + // When we download the new component shortly we will update our cache. + // This should only trigger after an app update and eventually this check can be removed. + return nil + } + + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( + source: source, + version: version + ) + + return AdBlockEngineManager.FileInfo(filterListInfo: filterListInfo, localFileURL: localFileURL) } /// Inform this manager of updates to the resources so our engines can be updated diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index d3d1856ad0e8..e3d06d3ecbb8 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -85,21 +85,15 @@ public actor FilterListResourceDownloader { folderURL: URL, engineType: GroupedAdBlockEngine.EngineType ) async { - let version = folderURL.lastPathComponent - let localFileURL = folderURL.appendingPathComponent("list.txt") - - guard FileManager.default.fileExists(atPath: localFileURL.relativePath) else { - // We are loading the old component from cache. We don't want this file to be loaded. - // When we download the new component shortly we will update our cache. - // This should only trigger after an app update and eventually this check can be removed. + guard + let fileInfo = await AdBlockGroupsManager.fileInfo( + for: source, + folderURL: folderURL + ) + else { return } - let fileInfo = AdBlockEngineManager.FileInfo( - filterListInfo: GroupedAdBlockEngine.FilterListInfo(source: source, version: version), - localFileURL: localFileURL - ) - await AdBlockGroupsManager.shared.updated( fileInfo: fileInfo, engineType: engineType diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift index 1cd3a018b43d..74d50ecd346b 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift @@ -299,6 +299,7 @@ extension FilterListStorage { return filterLists.isEmpty ? allFilterListSettings .filter(\.isEnabled) + .sorted(by: { $0.order?.intValue ?? 0 <= $1.order?.intValue ?? 0 }) .compactMap(\.engineSource) : filterLists .filter(\.isEnabled) From e08e35a23d38588d8084039ae2fb90deff01ea00 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Wed, 27 Mar 2024 22:24:53 +0100 Subject: [PATCH 04/10] Fix issue where content blockers are not updated --- .../Browser/Helpers/LaunchHelper.swift | 21 +- .../AdvancedShieldSettings.swift | 16 +- .../FilterLists/FilterListsView.swift | 4 +- .../AdBlock/AdBlockEngineManager.swift | 224 ++++++++++++------ .../AdBlock/AdBlockGroupsManager.swift | 114 +++++---- ...ockFilterListCatalogEntry+Extensions.swift | 30 +++ .../Sources/Brave/WebFilters/FilterList.swift | 15 -- .../Brave/WebFilters/FilterListStorage.swift | 32 +-- 8 files changed, 269 insertions(+), 187 deletions(-) create mode 100644 ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift index 045efb7a1d18..f783d89e2adc 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift @@ -50,14 +50,10 @@ public actor LaunchHelper { // This is done first because compileResources need their results await FilterListStorage.shared.loadFilterListSettings() await AdBlockGroupsManager.shared.loadResourcesFromCache() - - // Only load the standard engine for now, we will load the other one after launch - async let loadEngineFromCache: Void = AdBlockGroupsManager.shared.loadEngineFromCache( - for: .standard - ) + async let loadEngines: Void = AdBlockGroupsManager.shared.loadEnginesFromCache() async let adblockResourceCache: Void = AdblockResourceDownloader.shared .loadCachedAndBundledDataIfNeeded(allowedModes: launchBlockModes) - _ = await (loadEngineFromCache, adblockResourceCache) + _ = await (loadEngines, adblockResourceCache) Self.signpost.emitEvent("loadedCachedData", id: signpostID, "Loaded cached data") ContentBlockerManager.log.debug("Loaded blocking launch data") @@ -106,23 +102,10 @@ public actor LaunchHelper { let signpostID = Self.signpost.makeSignpostID() let state = Self.signpost.beginInterval("nonBlockingLaunchTask", id: signpostID) await FilterListResourceDownloader.shared.start(with: adBlockService) - Self.signpost.emitEvent( - "FilterListResourceDownloader.shared.start", - id: signpostID, - "Started filter list downloader" - ) await AdblockResourceDownloader.shared.loadCachedAndBundledDataIfNeeded( allowedModes: Set(remainingModes) ) - Self.signpost.emitEvent( - "loadCachedAndBundledDataIfNeeded", - id: signpostID, - "Reloaded data for remaining modes" - ) - await AdBlockGroupsManager.shared.loadEngineFromCache(for: .aggressive) await AdblockResourceDownloader.shared.startFetching() - Self.signpost.emitEvent("startFetching", id: signpostID, "Started fetching ad-block data") - /// Cleanup rule lists so we don't have dead rule lists let validBlocklistTypes = await self.getAllValidBlocklistTypes() await ContentBlockerManager.shared.cleaupInvalidRuleLists(validTypes: validBlocklistTypes) diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift index 35ce2f878e57..0d0a24e92057 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift @@ -28,17 +28,21 @@ import os @Published var cookieConsentBlocking: Bool { didSet { FilterListStorage.shared.ensureFilterList( - for: FilterList.cookieConsentNoticesComponentID, + for: AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID, isEnabled: cookieConsentBlocking ) + + AdBlockGroupsManager.shared.compileEnginesIfNeeded() } } @Published var blockMobileAnnoyances: Bool { didSet { FilterListStorage.shared.ensureFilterList( - for: FilterList.mobileAnnoyancesComponentID, + for: AdblockFilterListCatalogEntry.mobileAnnoyancesComponentID, isEnabled: blockMobileAnnoyances ) + + AdBlockGroupsManager.shared.compileEnginesIfNeeded() } } @Published var isP3AEnabled: Bool { @@ -101,11 +105,11 @@ import os self.isDebounceEnabled = debounceService?.isEnabled ?? false cookieConsentBlocking = FilterListStorage.shared.isEnabled( - for: FilterList.cookieConsentNoticesComponentID + for: AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID ) blockMobileAnnoyances = FilterListStorage.shared.isEnabled( - for: FilterList.mobileAnnoyancesComponentID + for: AdblockFilterListCatalogEntry.mobileAnnoyancesComponentID ) var clearableSettings = [ @@ -232,11 +236,11 @@ import os .sink { filterLists in for filterList in filterLists { switch filterList.entry.componentId { - case FilterList.cookieConsentNoticesComponentID: + case AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID: if filterList.isEnabled != self.cookieConsentBlocking { self.cookieConsentBlocking = filterList.isEnabled } - case FilterList.mobileAnnoyancesComponentID: + case AdblockFilterListCatalogEntry.mobileAnnoyancesComponentID: if filterList.isEnabled != self.blockMobileAnnoyances { self.blockMobileAnnoyances = filterList.isEnabled } diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift index c6bc2ab2c034..20ef128accd8 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift @@ -64,9 +64,7 @@ struct FilterListsView: View { ) } .onDisappear { - Task.detached { - await AdBlockGroupsManager.shared.ensureEnabledEngines() - } + AdBlockGroupsManager.shared.compileEnginesIfNeeded() } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index 8a026693e53d..1b053d0f6807 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -9,7 +9,7 @@ import Foundation import Preferences import os -/// A class for managing a grouped engine +/// A class for managing a single grouped engine @MainActor class AdBlockEngineManager { /// The directory to which we should store the unified list into private static var cacheFolderDirectory: FileManager.SearchPathDirectory { @@ -27,11 +27,18 @@ import os private var availableFiles: [FileInfo] /// The subfolder that the cache data is stored. /// Since we can have multiple engines this folder will be unique per engine - private var cacheFolderName: String + private let cacheFolderName: String /// The type of engine this represents what kind of blocking we are doing. (aggressive or standard) /// The difference is in how we treat first party blocking. Aggressive also blocks first party content. let engineType: GroupedAdBlockEngine.EngineType + /// The engine that is currently available for this manager private(set) var engine: GroupedAdBlockEngine? + /// This is a task that we use to delay a requested compile. + /// This allows us to wait for more sources and limit the number of times we compile + private var delayTask: Task? + /// This is the current pending group we are compiling. + /// This allows us to ensure we are not already compiling a newer version of the rules before setting the engine + private var pendingGroup: GroupedAdBlockEngine.FilterListGroup? /// This structure represents encodable info on what cached engine data contains private struct CachedEngineInfo: Codable { @@ -57,23 +64,6 @@ import os } } - /// Return an array of all sources that are enabled according to user's settings - /// - Note: This does not take into account the domain or global adblock toggle - private var enabledSources: [GroupedAdBlockEngine.Source] { - var enabledSources = FilterListStorage.shared.enabledSources(engineType: engineType) - if engineType.isAlwaysAggressive { - enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) - } - return enabledSources - } - - /// All the infos that are compilable based on the enabled sources and available infos - var compilableFiles: [FileInfo] { - return enabledSources.compactMap { source in - return availableFiles.first(where: { $0.filterListInfo.source == source }) - } - } - init(engineType: GroupedAdBlockEngine.EngineType, cacheFolderName: String) { self.availableFiles = [] self.engine = nil @@ -81,9 +71,13 @@ import os self.cacheFolderName = cacheFolderName } - /// Tells us if this source should be loaded. - func isEnabled(source: GroupedAdBlockEngine.Source) -> Bool { - return enabledSources.contains(source) + /// All the infos that are compilable based on the enabled sources and available infos + private func compilableFiles( + for enabledSources: [GroupedAdBlockEngine.Source] + ) -> [FileInfo] { + return enabledSources.compactMap { source in + return availableFiles.first(where: { $0.filterListInfo.source == source }) + } } /// Add the info to the available list @@ -104,22 +98,18 @@ import os } /// Checks to see if we need to compile or recompile the engine based on the available info - func checkNeedsCompile() -> Bool { - let compilableInfos = compilableFiles.map({ $0.filterListInfo }) + func checkNeedsCompile(for enabledSources: [GroupedAdBlockEngine.Source]) -> Bool { + let compilableInfos = compilableFiles(for: enabledSources).map({ $0.filterListInfo }) guard !compilableInfos.isEmpty else { return false } guard let engine = engine else { return true } return compilableInfos != engine.group.infos } - /// Checks to see if we need to compile or recompile the engine based on the available info - func checkHasAllInfo() -> Bool { - return compilableFiles == availableFiles - } - /// Load the engine from cache so it can be ready during launch func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async -> Bool { do { guard let cachedGroupInfo = loadCachedInfo() else { return false } + let engineType = self.engineType let groupedEngine = try await Task.detached(priority: .high) { let engine = try GroupedAdBlockEngine.compile( @@ -145,37 +135,128 @@ import os } } - /// This is a task that we use to delay a requested compile. - /// This allows us to wait for more sources and limit the number of times we compile - private var delayTask: Task? - /// This is the current pending group we are compiling. - /// This allows us to ensure we are not compiling a newer version of the rules - private var pendingGroup: GroupedAdBlockEngine.FilterListGroup? - /// This will compile available data, but will wait a little bit in case something new gets downloaded. /// Especially needed during launch when we have a bunch of downloads coming at the same time. - func compileDelayedIfNeeded(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) { + func compileDelayedIfNeeded( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, + priority: TaskPriority + ) { + guard self.checkNeedsCompile(for: enabledSources) else { return } + // Cancel the previous task delayTask?.cancel() // Restart the task delayTask = Task { - let hasAllInfo = checkHasAllInfo() - try await Task.sleep(seconds: hasAllInfo ? 10 : 60) - guard checkNeedsCompile() else { return } - try await compileAvailable(resourcesInfo: resourcesInfo) + let hasAllInfo = checkHasAllInfo(for: enabledSources) + try await Task.sleep(seconds: hasAllInfo ? 1 : 60) + await compileAvailableIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo, + priority: priority + ) + } + } + + /// This will compile available data right away if it is needed and cancel any delayedTasks + func compileImmediatelyIfNeeded( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, + priority: TaskPriority + ) { + delayTask?.cancel() + + Task { + await self.compileAvailableIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo, + priority: priority + ) + } + } + + /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. + func update(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) async { + do { + try await engine?.useResources(from: resourcesInfo) + } catch { + ContentBlockerManager.log.error( + "Failed to update engine resources for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + } + } + + /// Ensure all the content blockers are compiled for any file info found in the list of enabled sources + func ensureContentBlockers(for enabledSources: [GroupedAdBlockEngine.Source]) { + // Compile all content blockers for the given manager + compilableFiles(for: enabledSources).forEach { fileInfo in + Task { + await ensureContentBlockers(for: fileInfo) + } + } + } + + /// Ensure the content blocker is compiled for the given source + func ensureContentBlockers(for fileInfo: FileInfo) async { + guard + let blocklistType = fileInfo.filterListInfo.source.blocklistType( + isAlwaysAggressive: engineType.isAlwaysAggressive + ) + else { + return + } + + var modes = await ContentBlockerManager.shared.missingModes(for: blocklistType) + + if needsCompile(for: fileInfo.filterListInfo) { + modes = blocklistType.allowedModes + } + + await compileContentBlockers( + for: blocklistType, + localFileURL: fileInfo.localFileURL, + modes: modes + ) + } + + /// Checks to see if we need to compile or recompile the engine based on the available info + private func checkHasAllInfo(for sources: [GroupedAdBlockEngine.Source]) -> Bool { + return Set(sources) == Set(availableFiles.map(\.filterListInfo.source)) + } + + /// This will compile available data right away if it is needed + private func compileAvailableIfNeeded( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, + priority: TaskPriority + ) async { + do { + guard self.checkNeedsCompile(for: enabledSources) else { return } + try await compileAvailable( + for: enabledSources, + resourcesInfo: resourcesInfo, + priority: priority + ) + } catch { + ContentBlockerManager.log.error( + "Failed to compile engine for `\(self.cacheFolderName)`: \(String(describing: error))" + ) } } /// Compile an engine from all available data - func compileAvailable(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async throws { - // 1. Create the group - let group = try combineRules() + private func compileAvailable( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, + priority: TaskPriority + ) async throws { let engineType = self.engineType + let group = try combineRules(for: enabledSources) self.pendingGroup = group // 2. Compile the engine - let groupedEngine = try await Task.detached(priority: .background) { + let groupedEngine = try await Task.detached(priority: priority) { let engine = try GroupedAdBlockEngine.compile(group: group, type: engineType) if let resourcesInfo { @@ -203,46 +284,40 @@ import os self.engine = engine } - /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. - func update(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) async { - do { - try await engine?.useResources(from: resourcesInfo) - } catch { - ContentBlockerManager.log.error( - "Failed to update `\(self.cacheFolderName)` engne with resources" - ) + private func needsCompile(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) -> Bool { + guard let info = engine?.group.infos.first(where: { $0.source == filterListInfo.source }) else { + return true } + return filterListInfo.version < info.version } - func ensureContentBlockers(for fileInfo: FileInfo) async { - guard - let blocklistType = fileInfo.filterListInfo.source.blocklistType( - isAlwaysAggressive: engineType.isAlwaysAggressive - ) - else { - return - } - - let modes = await ContentBlockerManager.shared.missingModes(for: blocklistType) + /// Compile the content blockers for the given file info and modes + func compileContentBlockers( + for blocklistType: ContentBlockerManager.BlocklistType, + localFileURL: URL, + modes: [ContentBlockerManager.BlockingMode] + ) async { guard !modes.isEmpty else { return } do { try await ContentBlockerManager.shared.compileRuleList( - at: fileInfo.localFileURL, + at: localFileURL, for: blocklistType, modes: modes ) } catch { ContentBlockerManager.log.error( - "Failed to compile rule list for \(fileInfo.filterListInfo.debugDescription)" + "Failed to compile rule list for \(blocklistType.debugDescription)" ) } } /// Take all the filter lists and combine them into one then save them into a cache folder. - private func combineRules() throws -> GroupedAdBlockEngine.FilterListGroup { - // 1. Grab all the needed infos - let compilableFiles = compilableFiles + private func combineRules( + for enabledSources: [GroupedAdBlockEngine.Source] + ) throws -> GroupedAdBlockEngine.FilterListGroup { + // 1. Grab all the needed files + let compilableFiles = compilableFiles(for: enabledSources) // 2. Create a file url let cachedFolder = try getOrCreateCacheFolder() @@ -355,7 +430,7 @@ extension GroupedAdBlockEngine.Source { func blocklistType(isAlwaysAggressive: Bool) -> ContentBlockerManager.BlocklistType? { switch self { case .filterList(let componentId, let uuid): - guard uuid != FilterList.defaultComponentUUID else { + guard uuid != AdblockFilterListCatalogEntry.defaultFilterListComponentUUID else { // For now we don't compile this into content blockers because we use the one coming from slim list // We might change this in the future as it ends up with 95k items whereas the limit is 150k. // So there is really no reason to use slim list except perhaps for performance which we need to test out. @@ -369,19 +444,12 @@ extension GroupedAdBlockEngine.Source { } } -extension FilterListSetting { - @MainActor var engineSource: GroupedAdBlockEngine.Source? { - guard let componentId = componentId else { return nil } +extension AdblockFilterListCatalogEntry { + var engineSource: GroupedAdBlockEngine.Source { return .filterList(componentId: componentId, uuid: uuid) } } -extension FilterList { - @MainActor var engineSource: GroupedAdBlockEngine.Source { - return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) - } -} - extension CustomFilterListSetting { @MainActor var engineSource: GroupedAdBlockEngine.Source { return .filterListURL(uuid: uuid) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 45d5e31dde06..3f3f61e5fabf 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -9,8 +9,7 @@ import Foundation import Preferences import os -/// This object holds on to our adblock engines and returns information needed for stats tracking as well as some conveniences -/// for injected scripts needed during web navigation and cosmetic filters models needed by the `SelectorsPollerScript.js` script. +/// A class that helps manage file sources and enabled filter lists so that 2 engines (standard and aggressive) are continually updated. @MainActor public class AdBlockGroupsManager { typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) public static let shared = AdBlockGroupsManager( @@ -21,13 +20,17 @@ import os private let standardManager: AdBlockEngineManager private let aggressiveManager: AdBlockEngineManager - private var allManagers: [AdBlockEngineManager] { - return [standardManager, aggressiveManager] - } - /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. public var resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + /// Return an array of all sources that are enabled according to user's settings + /// - Note: This does not take into account the domain or global adblock toggle + private var enabledSources: [GroupedAdBlockEngine.Source] { + var enabledSources = FilterListStorage.shared.enabledSources + enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) + return enabledSources + } + init(standardManager: AdBlockEngineManager, aggressiveManager: AdBlockEngineManager) { self.standardManager = standardManager self.aggressiveManager = aggressiveManager @@ -44,21 +47,30 @@ import os func loadResourcesFromCache() async { if let resourcesFolderURL = FilterListSetting.makeFolderURL( forComponentFolderPath: Preferences.AppState.lastAdBlockResourcesFolderPath.value - ), FileManager.default.fileExists(atPath: resourcesFolderURL.path), - let resourcesInfo = getResourcesInfo(fromFolderURL: resourcesFolderURL) - { + ), FileManager.default.fileExists(atPath: resourcesFolderURL.path) { // We need this for all filter lists so we can't compile anything until we download it + let resourcesInfo = getResourcesInfo(fromFolderURL: resourcesFolderURL) self.resourcesInfo = resourcesInfo - + if #available(iOS 16.0, *) { ContentBlockerManager.log.debug( "Loaded resources component from cache: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" ) + } else { + ContentBlockerManager.log.debug( + "Loaded resources component from cache: `\(resourcesInfo.localFileURL.path)`" + ) } } } - func loadEngineFromCache(for engineType: GroupedAdBlockEngine.EngineType) async { + func loadEnginesFromCache() async { + await GroupedAdBlockEngine.EngineType.allCases.asyncConcurrentForEach { engineType in + await self.loadEngineFromCache(for: engineType) + } + } + + private func loadEngineFromCache(for engineType: GroupedAdBlockEngine.EngineType) async { let manager = getManager(for: engineType) if await !manager.loadFromCache(resourcesInfo: self.resourcesInfo) { @@ -75,13 +87,11 @@ import os manager.add(fileInfo: fileInfo) } - if manager.checkNeedsCompile() { - do { - try await manager.compileAvailable(resourcesInfo: self.resourcesInfo) - } catch { - - } - } + manager.compileImmediatelyIfNeeded( + for: enabledSources, + resourcesInfo: self.resourcesInfo, + priority: .high + ) } } @@ -129,39 +139,38 @@ import os engineType: GroupedAdBlockEngine.EngineType ) async { let manager = getManager(for: engineType) - // Always update the info on the manager - manager.add(fileInfo: fileInfo) - manager.compileDelayedIfNeeded(resourcesInfo: resourcesInfo) + let enabledSources = enabledSources // Compile content blockers if this filter list is enabled - if manager.isEnabled(source: fileInfo.filterListInfo.source) { + if enabledSources.contains(fileInfo.filterListInfo.source) { await manager.ensureContentBlockers(for: fileInfo) } - } - /// Ensure all engines and content blockers are compiled - func ensureEnabledEngines() async { - guard let resourcesInfo = self.resourcesInfo else { return } + // Always update the info on the manager + manager.add(fileInfo: fileInfo) + manager.compileDelayedIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo, + priority: .background + ) - await allManagers.asyncConcurrentForEach { manager in - // Compile engines - do { - if manager.checkNeedsCompile() { - try await manager.compileAvailable(resourcesInfo: resourcesInfo) - } - } catch { - // Ignore cancellation errors - } + } - // Compile all content blockers for the given manager - await manager.compilableFiles.asyncForEach { fileInfo in - guard manager.isEnabled(source: fileInfo.filterListInfo.source) else { return } - await manager.ensureContentBlockers(for: fileInfo) - } + /// Ensure all engines and content blockers are compiled + func compileEnginesIfNeeded() { + let enabledSources = enabledSources + GroupedAdBlockEngine.EngineType.allCases.forEach { engineType in + let manager = self.getManager(for: engineType) + manager.compileImmediatelyIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo, + priority: .high + ) + manager.ensureContentBlockers(for: enabledSources) } } - private func getResourcesInfo(fromFolderURL folderURL: URL) -> GroupedAdBlockEngine.ResourcesInfo? + private func getResourcesInfo(fromFolderURL folderURL: URL) -> GroupedAdBlockEngine.ResourcesInfo { let version = folderURL.lastPathComponent return GroupedAdBlockEngine.ResourcesInfo( @@ -177,7 +186,9 @@ import os } self.resourcesInfo = resourcesInfo - allManagers.forEach { manager in + GroupedAdBlockEngine.EngineType.allCases.forEach { engineType in + let manager = self.getManager(for: engineType) + Task { await manager.update(resourcesInfo: resourcesInfo) } @@ -187,6 +198,10 @@ import os ContentBlockerManager.log.debug( "Updated resources component: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" ) + } else { + ContentBlockerManager.log.debug( + "Updated resources component: `\(resourcesInfo.localFileURL.path)`" + ) } } @@ -241,7 +256,7 @@ import os /// Returns all appropriate engines for the given domain func cachedEngines(for domain: Domain) -> [GroupedAdBlockEngine] { guard domain.isShieldExpected(.adblockAndTp, considerAllShieldsOption: true) else { return [] } - return allManagers.compactMap({ $0.engine }) + return GroupedAdBlockEngine.EngineType.allCases.compactMap({ getManager(for: $0).engine }) } /// Returns all the models for this frame URL @@ -283,3 +298,16 @@ extension GroupedAdBlockEngine.EngineType { return AdBlockEngineManager(engineType: self, cacheFolderName: defaultCachedFolderName) } } + +extension FilterListSetting { + @MainActor var engineSource: GroupedAdBlockEngine.Source? { + guard let componentId = componentId else { return nil } + return .filterList(componentId: componentId, uuid: uuid) + } +} + +extension FilterList { + @MainActor var engineSource: GroupedAdBlockEngine.Source { + return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift new file mode 100644 index 000000000000..6614354fd66b --- /dev/null +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift @@ -0,0 +1,30 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Foundation + +extension AdblockFilterListCatalogEntry { + /// This is the uuid of the default filter list. This is a special filter list which we use slim list content blockers for + public static let defaultFilterListComponentUUID = "default" + /// The component ID of the "Fanboy's Mobile Notifications List" + /// This is a special filter list that is enabled by default + public static let mobileAnnoyancesComponentID = "bfpgedeaaibpoidldhjcknekahbikncb" + /// The component id of the cookie consent notices filter list. + /// This is a special filter list that has more accessible UI to control it + public static let cookieConsentNoticesComponentID = "cdbbhgbmjhfnhnmgeddbliobbofkgdhe" + + public static let disabledFilterListComponentIDs = [ + // The Anti-porn list has 500251 rules and is strictly all content blocking driven content + // The limit for the rule store is 150000 rules. We have no way to handle this at the current moment + "lbnibkdpkdjnookgfeogjdanfenekmpe" + ] + + /// Lets us know if this filter list is always aggressive. + /// This value comes from `list_catalog.json` in brave core + var engineType: GroupedAdBlockEngine.EngineType { + return firstPartyProtections ? .standard : .aggressive + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift index 145d99b628d6..feb1110e3437 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift @@ -7,21 +7,6 @@ import BraveCore import Foundation struct FilterList: Identifiable { - /// This is the uuid of the default filter list. This is a special filter list which we use slim list content blockers for - public static let defaultComponentUUID = "default" - /// The component ID of the "Fanboy's Mobile Notifications List" - /// This is a special filter list that is enabled by default - public static let mobileAnnoyancesComponentID = "bfpgedeaaibpoidldhjcknekahbikncb" - /// The component id of the cookie consent notices filter list. - /// This is a special filter list that has more accessible UI to control it - public static let cookieConsentNoticesComponentID = "cdbbhgbmjhfnhnmgeddbliobbofkgdhe" - /// This is a list of disabled filter lists. These lists are disabled because they are incompatible with iOS (for the time being) - public static let disabledComponentIDs = [ - // The Anti-porn list has 500251 rules and is strictly all content blocking driven content - // The limit for the rule store is 150000 rules. We have no way to handle this at the current moment - "lbnibkdpkdjnookgfeogjdanfenekmpe" - ] - var id: String { return entry.uuid } let order: Int let entry: AdblockFilterListCatalogEntry diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift index 74d50ecd346b..148d237aea0f 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift @@ -56,7 +56,11 @@ import Preferences index, adBlockFilterList -> FilterList? in // Certain filter lists are disabled if they are currently incompatible with iOS - guard !FilterList.disabledComponentIDs.contains(adBlockFilterList.componentId) else { + guard + !AdblockFilterListCatalogEntry.disabledFilterListComponentIDs.contains( + adBlockFilterList.componentId + ) + else { return nil } let setting = allFilterListSettings.first(where: { @@ -125,7 +129,9 @@ import Preferences /// - Warning: Do not call this before we load core data public func isEnabled(for componentId: String) -> Bool { - guard !FilterList.disabledComponentIDs.contains(componentId) else { return false } + guard !AdblockFilterListCatalogEntry.disabledFilterListComponentIDs.contains(componentId) else { + return false + } return filterLists.first(where: { $0.entry.componentId == componentId })?.isEnabled ?? allFilterListSettings.first(where: { $0.componentId == componentId })?.isEnabled @@ -276,7 +282,7 @@ import Preferences Task { @MainActor in UmaHistogramBoolean( "Brave.Shields.CookieListEnabled", - isEnabled(for: FilterList.cookieConsentNoticesComponentID) + isEnabled(for: AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID) ) } } @@ -305,24 +311,4 @@ extension FilterListStorage { .filter(\.isEnabled) .map(\.engineSource) } - - /// Gives us source representations of all the enabled filter lists - @MainActor func enabledSources( - engineType: GroupedAdBlockEngine.EngineType - ) -> [GroupedAdBlockEngine.Source] { - if !filterLists.isEmpty { - return filterLists.compactMap { filterList -> GroupedAdBlockEngine.Source? in - guard filterList.engineType == engineType else { return nil } - guard filterList.isEnabled else { return nil } - return filterList.engineSource - } - } else { - // We may not have the filter lists loaded yet. In which case we load the settings - return allFilterListSettings.compactMap { setting -> GroupedAdBlockEngine.Source? in - guard setting.isAlwaysAggressive == engineType.isAlwaysAggressive else { return nil } - guard setting.isEnabled else { return nil } - return setting.engineSource - } - } - } } From a73ec41fc78c84a1a11989b41dbe743b1ccf8a08 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Thu, 28 Mar 2024 12:05:19 +0100 Subject: [PATCH 05/10] Add tests --- .../AdvancedShieldSettings.swift | 8 +- .../FilterLists/FilterListsView.swift | 4 +- .../AdBlock/AdBlockEngineManager.swift | 84 ++---- .../AdBlock/AdBlockGroupsManager.swift | 182 ++++++++---- .../AdBlock/GroupedAdBlockEngine.swift | 1 - .../ContentBlockerManager.swift | 28 +- .../FilterListResourceDownloader.swift | 4 +- .../AdBlockEngineManagerTests.swift | 121 ++++++++ .../AdBlockGroupsManagerTests.swift | 273 ++++++++++++++++++ .../Tests/ClientTests/PageDataTests.swift | 41 ++- .../GroupedAdBlockEngineTests.swift | 12 +- 11 files changed, 601 insertions(+), 157 deletions(-) create mode 100644 ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift create mode 100644 ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift index 0d0a24e92057..210a30bd0f0f 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift @@ -32,7 +32,9 @@ import os isEnabled: cookieConsentBlocking ) - AdBlockGroupsManager.shared.compileEnginesIfNeeded() + Task { + await AdBlockGroupsManager.shared.compileEnginesIfNeeded() + } } } @Published var blockMobileAnnoyances: Bool { @@ -42,7 +44,9 @@ import os isEnabled: blockMobileAnnoyances ) - AdBlockGroupsManager.shared.compileEnginesIfNeeded() + Task { + await AdBlockGroupsManager.shared.compileEnginesIfNeeded() + } } } @Published var isP3AEnabled: Bool { diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift index 20ef128accd8..3d577fe98df6 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift @@ -64,7 +64,9 @@ struct FilterListsView: View { ) } .onDisappear { - AdBlockGroupsManager.shared.compileEnginesIfNeeded() + Task { + await AdBlockGroupsManager.shared.compileEnginesIfNeeded() + } } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index 1b053d0f6807..6cbe353c641a 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -72,7 +72,7 @@ import os } /// All the infos that are compilable based on the enabled sources and available infos - private func compilableFiles( + func compilableFiles( for enabledSources: [GroupedAdBlockEngine.Source] ) -> [FileInfo] { return enabledSources.compactMap { source in @@ -135,6 +135,12 @@ import os } } + // Delete the cache. Mostly used for testing + func deleteCachedEngine() throws { + guard let cacheFolderURL = createdCacheFolderURL else { return } + try FileManager.default.removeItem(at: cacheFolderURL) + } + /// This will compile available data, but will wait a little bit in case something new gets downloaded. /// Especially needed during launch when we have a bunch of downloads coming at the same time. func compileDelayedIfNeeded( @@ -164,16 +170,14 @@ import os for enabledSources: [GroupedAdBlockEngine.Source], resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, priority: TaskPriority - ) { + ) async { delayTask?.cancel() - Task { - await self.compileAvailableIfNeeded( - for: enabledSources, - resourcesInfo: resourcesInfo, - priority: priority - ) - } + await self.compileAvailableIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo, + priority: priority + ) } /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. @@ -187,37 +191,11 @@ import os } } - /// Ensure all the content blockers are compiled for any file info found in the list of enabled sources - func ensureContentBlockers(for enabledSources: [GroupedAdBlockEngine.Source]) { - // Compile all content blockers for the given manager - compilableFiles(for: enabledSources).forEach { fileInfo in - Task { - await ensureContentBlockers(for: fileInfo) - } - } - } - - /// Ensure the content blocker is compiled for the given source - func ensureContentBlockers(for fileInfo: FileInfo) async { - guard - let blocklistType = fileInfo.filterListInfo.source.blocklistType( - isAlwaysAggressive: engineType.isAlwaysAggressive - ) - else { - return - } - - var modes = await ContentBlockerManager.shared.missingModes(for: blocklistType) - - if needsCompile(for: fileInfo.filterListInfo) { - modes = blocklistType.allowedModes + func needsCompile(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) -> Bool { + guard let info = engine?.group.infos.first(where: { $0.source == filterListInfo.source }) else { + return true } - - await compileContentBlockers( - for: blocklistType, - localFileURL: fileInfo.localFileURL, - modes: modes - ) + return filterListInfo.version < info.version } /// Checks to see if we need to compile or recompile the engine based on the available info @@ -284,34 +262,6 @@ import os self.engine = engine } - private func needsCompile(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) -> Bool { - guard let info = engine?.group.infos.first(where: { $0.source == filterListInfo.source }) else { - return true - } - return filterListInfo.version < info.version - } - - /// Compile the content blockers for the given file info and modes - func compileContentBlockers( - for blocklistType: ContentBlockerManager.BlocklistType, - localFileURL: URL, - modes: [ContentBlockerManager.BlockingMode] - ) async { - guard !modes.isEmpty else { return } - - do { - try await ContentBlockerManager.shared.compileRuleList( - at: localFileURL, - for: blocklistType, - modes: modes - ) - } catch { - ContentBlockerManager.log.error( - "Failed to compile rule list for \(blocklistType.debugDescription)" - ) - } - } - /// Take all the filter lists and combine them into one then save them into a cache folder. private func combineRules( for enabledSources: [GroupedAdBlockEngine.Source] diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 3f3f61e5fabf..1e7faf5993c4 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -11,29 +11,45 @@ import os /// A class that helps manage file sources and enabled filter lists so that 2 engines (standard and aggressive) are continually updated. @MainActor public class AdBlockGroupsManager { + @MainActor protocol SourceProvider { + /// All the enabled sources. + /// They will be compiled in the order given so ensure the order here corresponds with how you want your engines to look + var enabledSources: [GroupedAdBlockEngine.Source] { get } + + /// If we didn't load the main engine from cache we need to load using the old cache mechanism + /// This is only temporary so we're not left with no ad-block during the upgrade. + /// We can drop all of this in future upgrades as by then we will have files cached in the new format + func legacyCacheFiles( + for engineType: GroupedAdBlockEngine.EngineType + ) -> [AdBlockEngineManager.FileInfo] + } + typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) public static let shared = AdBlockGroupsManager( standardManager: GroupedAdBlockEngine.EngineType.standard.makeDefaultManager(), - aggressiveManager: GroupedAdBlockEngine.EngineType.aggressive.makeDefaultManager() + aggressiveManager: GroupedAdBlockEngine.EngineType.aggressive.makeDefaultManager(), + contentBlockerManager: ContentBlockerManager.shared, + sourceProvider: DefaultSourceProvider() ) private let standardManager: AdBlockEngineManager private let aggressiveManager: AdBlockEngineManager + private let contentBlockerManager: ContentBlockerManager + private let sourceProvider: SourceProvider /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. public var resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? - /// Return an array of all sources that are enabled according to user's settings - /// - Note: This does not take into account the domain or global adblock toggle - private var enabledSources: [GroupedAdBlockEngine.Source] { - var enabledSources = FilterListStorage.shared.enabledSources - enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) - return enabledSources - } - - init(standardManager: AdBlockEngineManager, aggressiveManager: AdBlockEngineManager) { + init( + standardManager: AdBlockEngineManager, + aggressiveManager: AdBlockEngineManager, + contentBlockerManager: ContentBlockerManager, + sourceProvider: SourceProvider + ) { self.standardManager = standardManager self.aggressiveManager = aggressiveManager + self.contentBlockerManager = contentBlockerManager + self.sourceProvider = sourceProvider self.resourcesInfo = nil } @@ -74,49 +90,18 @@ import os let manager = getManager(for: engineType) if await !manager.loadFromCache(resourcesInfo: self.resourcesInfo) { - // If we didn't load the main engine from cache we need to load using the old cache mechanism - // This is only temporary so we're not left with no ad-block during the upgrade. - // We can drop all of this in future upgrades as by then we will have files cached in the new format - for setting in FilterListStorage.shared.allFilterListSettings - .filter({ $0.isAlwaysAggressive == engineType.isAlwaysAggressive }) - .sorted(by: { $0.order?.intValue ?? 0 <= $1.order?.intValue ?? 0 }) - { - guard let folderURL = setting.folderURL else { continue } - guard let source = setting.engineSource else { continue } - guard let fileInfo = Self.fileInfo(for: source, folderURL: folderURL) else { continue } + for fileInfo in sourceProvider.legacyCacheFiles(for: engineType) { manager.add(fileInfo: fileInfo) } - manager.compileImmediatelyIfNeeded( - for: enabledSources, + await manager.compileImmediatelyIfNeeded( + for: sourceProvider.enabledSources, resourcesInfo: self.resourcesInfo, priority: .high ) } } - static func fileInfo( - for source: GroupedAdBlockEngine.Source, - folderURL: URL - ) -> AdBlockEngineManager.FileInfo? { - let version = folderURL.lastPathComponent - let localFileURL = folderURL.appendingPathComponent("list.txt") - - guard FileManager.default.fileExists(atPath: localFileURL.relativePath) else { - // We are loading the old component from cache. We don't want this file to be loaded. - // When we download the new component shortly we will update our cache. - // This should only trigger after an app update and eventually this check can be removed. - return nil - } - - let filterListInfo = GroupedAdBlockEngine.FilterListInfo( - source: source, - version: version - ) - - return AdBlockEngineManager.FileInfo(filterListInfo: filterListInfo, localFileURL: localFileURL) - } - /// Inform this manager of updates to the resources so our engines can be updated func didUpdateResourcesComponent(folderURL: URL) async { await Task { @MainActor in @@ -139,11 +124,11 @@ import os engineType: GroupedAdBlockEngine.EngineType ) async { let manager = getManager(for: engineType) - let enabledSources = enabledSources + let enabledSources = sourceProvider.enabledSources // Compile content blockers if this filter list is enabled if enabledSources.contains(fileInfo.filterListInfo.source) { - await manager.ensureContentBlockers(for: fileInfo) + await ensureContentBlockers(for: fileInfo, engineType: engineType) } // Always update the info on the manager @@ -157,19 +142,62 @@ import os } /// Ensure all engines and content blockers are compiled - func compileEnginesIfNeeded() { - let enabledSources = enabledSources - GroupedAdBlockEngine.EngineType.allCases.forEach { engineType in + func compileEnginesIfNeeded() async { + let enabledSources = sourceProvider.enabledSources + await GroupedAdBlockEngine.EngineType.allCases.asyncConcurrentForEach { engineType in let manager = self.getManager(for: engineType) - manager.compileImmediatelyIfNeeded( + await manager.compileImmediatelyIfNeeded( for: enabledSources, - resourcesInfo: resourcesInfo, + resourcesInfo: self.resourcesInfo, priority: .high ) - manager.ensureContentBlockers(for: enabledSources) + + self.ensureContentBlockers(for: enabledSources, engineType: engineType) + } + } + + /// Ensure all the content blockers are compiled for any file info found in the list of enabled sources + private func ensureContentBlockers( + for enabledSources: [GroupedAdBlockEngine.Source], + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + // Compile all content blockers for the given manager + manager.compilableFiles(for: enabledSources).forEach { fileInfo in + Task { + await ensureContentBlockers(for: fileInfo, engineType: engineType) + } } } + /// Ensure the content blocker is compiled for the given source + private func ensureContentBlockers( + for fileInfo: AdBlockEngineManager.FileInfo, + engineType: GroupedAdBlockEngine.EngineType + ) async { + let manager = getManager(for: engineType) + + guard + let blocklistType = fileInfo.filterListInfo.source.blocklistType( + isAlwaysAggressive: engineType.isAlwaysAggressive + ) + else { + return + } + + var modes = await contentBlockerManager.missingModes(for: blocklistType) + + if manager.needsCompile(for: fileInfo.filterListInfo) { + modes = blocklistType.allowedModes + } + + await contentBlockerManager.compileRuleList( + at: fileInfo.localFileURL, + for: blocklistType, + modes: modes + ) + } + private func getResourcesInfo(fromFolderURL folderURL: URL) -> GroupedAdBlockEngine.ResourcesInfo { let version = folderURL.lastPathComponent @@ -180,7 +208,7 @@ import os } /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. - private func updateIfNeeded(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) { + func updateIfNeeded(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) { guard self.resourcesInfo == nil || resourcesInfo.version > self.resourcesInfo!.version else { return } @@ -237,7 +265,6 @@ import os return try await cachedEngine.makeEngineScriptTypes( frameURL: frameURL, isMainFrame: isMainFrame, - domain: domain, isDeAmpEnabled: isDeAmpEnabled, index: index ) @@ -311,3 +338,50 @@ extension FilterList { return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) } } + +extension AdBlockEngineManager.FileInfo { + init?( + for source: GroupedAdBlockEngine.Source, + downloadedFolderURL: URL + ) { + let version = downloadedFolderURL.lastPathComponent + let localFileURL = downloadedFolderURL.appendingPathComponent("list.txt") + + guard FileManager.default.fileExists(atPath: localFileURL.relativePath) else { + // We are loading the old component from cache. We don't want this file to be loaded. + // When we download the new component shortly we will update our cache. + // This should only trigger after an app update and eventually this check can be removed. + return nil + } + + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( + source: source, + version: version + ) + + self.init(filterListInfo: filterListInfo, localFileURL: localFileURL) + } +} + +@MainActor private class DefaultSourceProvider: AdBlockGroupsManager.SourceProvider { + /// Return an array of all sources that are enabled according to user's settings + /// - Note: This does not take into account the domain or global adblock toggle + var enabledSources: [GroupedAdBlockEngine.Source] { + var enabledSources = FilterListStorage.shared.enabledSources + enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) + return enabledSources + } + + func legacyCacheFiles( + for engineType: GroupedAdBlockEngine.EngineType + ) -> [AdBlockEngineManager.FileInfo] { + return FilterListStorage.shared.allFilterListSettings + .filter({ $0.isAlwaysAggressive == engineType.isAlwaysAggressive }) + .sorted(by: { $0.order?.intValue ?? 0 <= $1.order?.intValue ?? 0 }) + .compactMap({ setting in + guard let folderURL = setting.folderURL else { return nil } + guard let source = setting.engineSource else { return nil } + return AdBlockEngineManager.FileInfo(for: source, downloadedFolderURL: folderURL) + }) + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift index ae0c601eb489..21ad0951d8bc 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift @@ -154,7 +154,6 @@ public actor GroupedAdBlockEngine { func makeEngineScriptTypes( frameURL: URL, isMainFrame: Bool, - domain: Domain, isDeAmpEnabled: Bool, index: Int ) throws -> Set { diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift index 1c22a83f62b8..416c7fd79e6e 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift @@ -184,27 +184,35 @@ actor ContentBlockerManager { /// Compile the rule list found in the given local URL using the specified modes func compileRuleList( at localFileURL: URL, - for type: BlocklistType, + for blocklistType: BlocklistType, modes: [BlockingMode] - ) async throws { + ) async { + guard !modes.isEmpty else { return } + let result: ContentBlockingRulesResult let signpostID = Self.signpost.makeSignpostID() let state = Self.signpost.beginInterval( "convertRules", id: signpostID, - "\(type.debugDescription)" + "\(blocklistType.debugDescription)" ) do { - let filterSet = try String(contentsOf: localFileURL) - result = try AdblockEngine.contentBlockerRules(fromFilterSet: filterSet) - Self.signpost.endInterval("convertRules", state) + do { + let filterSet = try String(contentsOf: localFileURL) + result = try AdblockEngine.contentBlockerRules(fromFilterSet: filterSet) + Self.signpost.endInterval("convertRules", state) + } catch { + Self.signpost.endInterval("convertRules", state, "\(error.localizedDescription)") + throw error + } + + try await compile(encodedContentRuleList: result.rulesJSON, for: blocklistType, modes: modes) } catch { - Self.signpost.endInterval("convertRules", state, "\(error.localizedDescription)") - throw error + ContentBlockerManager.log.error( + "Failed to compile rule list for \(blocklistType.debugDescription)" + ) } - - try await compile(encodedContentRuleList: result.rulesJSON, for: type, modes: modes) } /// Compile the given resource and store it in cache for the given blocklist type and specified modes diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index e3d06d3ecbb8..5daf32a3f5f8 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -86,9 +86,9 @@ public actor FilterListResourceDownloader { engineType: GroupedAdBlockEngine.EngineType ) async { guard - let fileInfo = await AdBlockGroupsManager.fileInfo( + let fileInfo = AdBlockEngineManager.FileInfo( for: source, - folderURL: folderURL + downloadedFolderURL: folderURL ) else { return diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift new file mode 100644 index 000000000000..470298bb7848 --- /dev/null +++ b/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift @@ -0,0 +1,121 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import XCTest + +@testable import Brave + +final class AdBlockEngineManagerTests: XCTestCase { + func testEngineManager() async throws { + // Given + // Engine manager and file info and resources info + let engineManager = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test-standard" + ) + let sampleFilterListURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", + withExtension: "txt" + )! + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + version: "0" + ) + let fileInfos = [ + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: sampleFilterListURL + ), + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: sampleFilterListURL + ), + ] + let sources = fileInfos.map({ $0.filterListInfo.source }) + + // When + // Adding file info + for fileInfo in fileInfos { + await engineManager.add(fileInfo: fileInfo) + } + + // Then + // Needs compile returns true and there is no engine + var needsCompile = await engineManager.checkNeedsCompile(for: sources) + var engine = await engineManager.engine + XCTAssertTrue(needsCompile) + XCTAssertNil(engine) + + // When + // Loaded from cache + var loadedFromCache = await engineManager.loadFromCache(resourcesInfo: resourcesInfo) + + // Then + // We don't have any engine + engine = await engineManager.engine + XCTAssertNil(engine) + XCTAssertFalse(loadedFromCache) + + // When 2 + // We compile engine + await engineManager.compileImmediatelyIfNeeded( + for: sources, + resourcesInfo: resourcesInfo, + priority: .high + ) + + // Then + // Needs compile returns false and engine is correctly created + needsCompile = await engineManager.checkNeedsCompile(for: sources) + engine = await engineManager.engine + let compiledResources = await engine?.resourcesInfo + let group = await engine?.group + XCTAssertFalse(needsCompile) + XCTAssertNotNil(engine) + XCTAssertEqual(group?.infos, fileInfos.map({ $0.filterListInfo })) + XCTAssertEqual(group?.fileType, .text) + XCTAssertEqual(compiledResources, resourcesInfo) + + // When + // We load from cache using another manager with the same cache folder name + let engineManager2 = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test-standard" + ) + loadedFromCache = await engineManager2.loadFromCache(resourcesInfo: resourcesInfo) + + // Then + // We have a cached engine + engine = await engineManager2.engine + let cacheGroup = await engine?.group + let expectedURL = group?.localFileURL.deletingPathExtension().appendingPathExtension("dat") + XCTAssertTrue(loadedFromCache) + XCTAssertNotNil(engine) + XCTAssertEqual(cacheGroup?.localFileURL, expectedURL) + XCTAssertEqual(cacheGroup?.infos, fileInfos.map({ $0.filterListInfo })) + XCTAssertEqual(cacheGroup?.fileType, .data) + try await engineManager2.deleteCachedEngine() + + // When + // We load from cache using a third manager with the same cache folder name + let engineManager3 = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test-standard" + ) + loadedFromCache = await engineManager3.loadFromCache(resourcesInfo: resourcesInfo) + + // Then + // There is no cached engine because we deleted it previously + engine = await engineManager3.engine + XCTAssertFalse(loadedFromCache) + XCTAssertNil(engine) + } +} diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift new file mode 100644 index 000000000000..27d358d1473c --- /dev/null +++ b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift @@ -0,0 +1,273 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Data +import WebKit +import XCTest + +@testable import Brave + +final class AdBlockGroupsManagerTests: XCTestCase { + private static var sampleFilterListURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", + withExtension: "txt" + )! + + @MainActor private lazy var ruleStore: WKContentRuleListStore = { + let testBundle = Bundle.module + let bundleURL = testBundle.bundleURL + return WKContentRuleListStore(url: bundleURL)! + }() + + /// Testing engine compilations and filter list managment + func testCompilation() async throws { + AdblockEngine.setDomainResolver() + + // Given + // A source provider and groups manager + let fileInfos = [ + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: Self.sampleFilterListURL + ), + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: Self.sampleFilterListURL + ), + ] + let sourceProvider = await TestSourceProvider(fileInfos: fileInfos) + let standardManager = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ) + let aggressiveManager = await AdBlockEngineManager( + engineType: .aggressive, + cacheFolderName: "test_aggressive" + ) + let groupsManager = await AdBlockGroupsManager( + standardManager: standardManager, + aggressiveManager: aggressiveManager, + contentBlockerManager: makeContentBlockingManager(), + sourceProvider: sourceProvider + ) + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + version: "0" + ) + + // When + // Adding file info and enabling sources for only one engine + await sourceProvider.set( + source: fileInfos[0].filterListInfo.source, + enabled: true + ) + await groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + await groupsManager.updated(fileInfo: fileInfos[1], engineType: .standard) + await groupsManager.updated(fileInfo: fileInfos[0], engineType: .aggressive) + await groupsManager.updated(fileInfo: fileInfos[1], engineType: .aggressive) + await groupsManager.compileEnginesIfNeeded() + + // Then + // Only one engine is created + var standardEngine = await standardManager.engine + var aggressiveEngine = await aggressiveManager.engine + var standardGroup = await standardEngine?.group + var aggressiveGroup = await aggressiveEngine?.group + var standardResources = await standardEngine?.resourcesInfo + var aggressiveResources = await aggressiveEngine?.resourcesInfo + XCTAssertNil(standardEngine) + XCTAssertNotNil(aggressiveEngine) + XCTAssertEqual(aggressiveResources, resourcesInfo) + XCTAssertEqual(aggressiveGroup?.infos, [fileInfos[0].filterListInfo]) + + // When + // We enable sources and recompile the engine + await sourceProvider.set( + source: fileInfos[1].filterListInfo.source, + enabled: true + ) + await groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + await groupsManager.compileEnginesIfNeeded() + + // Then + // All engines are created + standardEngine = await standardManager.engine + aggressiveEngine = await aggressiveManager.engine + standardGroup = await standardEngine?.group + aggressiveGroup = await aggressiveEngine?.group + standardResources = await standardEngine?.resourcesInfo + aggressiveResources = await aggressiveEngine?.resourcesInfo + XCTAssertNotNil(standardEngine) + XCTAssertNotNil(aggressiveEngine) + XCTAssertEqual(standardResources, resourcesInfo) + XCTAssertEqual(aggressiveResources, resourcesInfo) + XCTAssertEqual(standardGroup?.infos, [fileInfos[1].filterListInfo]) + XCTAssertEqual(aggressiveGroup?.infos, fileInfos.map({ $0.filterListInfo })) + } + + /// Partially testing expectations found in https://dev-pages.bravesoftware.com/filtering/index.html + @MainActor func testBlocking() async throws { + AdblockEngine.setDomainResolver() + + // Given + // A source provider and groups manager + let fileInfo = AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: Self.sampleFilterListURL + ) + let sourceProvider = TestSourceProvider(fileInfos: [fileInfo]) + let standardManager = AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ) + let aggressiveManager = AdBlockEngineManager( + engineType: .aggressive, + cacheFolderName: "test_aggressive" + ) + let groupsManager = AdBlockGroupsManager( + standardManager: standardManager, + aggressiveManager: aggressiveManager, + contentBlockerManager: makeContentBlockingManager(), + sourceProvider: sourceProvider + ) + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + version: "0" + ) + + // When + // Adding creating a standard engine + sourceProvider.set( + source: fileInfo.filterListInfo.source, + enabled: true + ) + groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + await groupsManager.updated(fileInfo: fileInfo, engineType: .standard) + await groupsManager.compileEnginesIfNeeded() + + let mainFrameURL = URL(string: "https://dev-pages.bravesoftware.com")! + let domain = await MainActor.run { + return Domain.getOrCreate(forUrl: mainFrameURL, persistent: false) + } + + // Then + // Should have certian script types + let scriptTypes = await groupsManager.makeEngineScriptTypes( + frameURL: URL(string: "https://dev-pages.bravesoftware.com/filtering/scriptlets.html")!, + isMainFrame: true, + isDeAmpEnabled: false, + domain: domain + ) + XCTAssertTrue( + scriptTypes.contains(where: { scriptType in + switch scriptType { + case .engineScript(let configuration): + return configuration.source.contains("window.BRAVE_TEST_VALUE") + default: + return false + } + }) + ) + + // Then + // Should return some filter models + let cosmeticFilterModels = await groupsManager.cosmeticFilterModels( + forFrameURL: mainFrameURL, + domain: domain + ) + XCTAssertFalse(cosmeticFilterModels.isEmpty) + + // Should block certain 3rd party content + let sourceURL = URL( + string: "https://dev-pages.bravesoftware.com/filtering/network-requests.html" + )! + let requestURL = URL( + string: "https://dev-pages.brave.software/static/images/test.jpg?335962573013224749" + )! + var blockResult = await groupsManager.shouldBlock( + requestURL: requestURL, + sourceURL: sourceURL, + resourceType: .image, + domain: domain + ) + XCTAssertTrue(blockResult) + + // Should not block certain 1sd party content + let requestURL2 = URL( + string: "https://dev-pages.bravesoftware.com/static/images/test.jpg?335962573013224749" + )! + blockResult = await groupsManager.shouldBlock( + requestURL: requestURL2, + sourceURL: sourceURL, + resourceType: .image, + domain: domain + ) + XCTAssertFalse(blockResult) + + // When + // Adding an aggressive filter list + await groupsManager.updated(fileInfo: fileInfo, engineType: .aggressive) + await groupsManager.compileEnginesIfNeeded() + + // Then + // Should block certain 1sd party content + blockResult = await groupsManager.shouldBlock( + requestURL: requestURL2, + sourceURL: sourceURL, + resourceType: .image, + domain: domain + ) + XCTAssertTrue(blockResult) + } + + @MainActor private func makeContentBlockingManager() -> ContentBlockerManager { + return ContentBlockerManager(ruleStore: ruleStore) + } +} + +class TestSourceProvider: AdBlockGroupsManager.SourceProvider { + /// The sources that are enabled + private var _enabledSources: Set + private var fileInfos: [AdBlockEngineManager.FileInfo] = [] + + init( + _enabledSources: Set = [], + fileInfos: [AdBlockEngineManager.FileInfo] = [] + ) { + self._enabledSources = _enabledSources + self.fileInfos = fileInfos + } + + func set(source: GroupedAdBlockEngine.Source, enabled: Bool) { + if enabled { + _enabledSources.insert(source) + } else { + _enabledSources.remove(source) + } + } + + var enabledSources: [GroupedAdBlockEngine.Source] { + return fileInfos.compactMap { fileInfo in + guard _enabledSources.contains(fileInfo.filterListInfo.source) else { return nil } + return fileInfo.filterListInfo.source + } + } + + func legacyCacheFiles( + for engineType: Brave.GroupedAdBlockEngine.EngineType + ) -> [Brave.AdBlockEngineManager.FileInfo] { + return [] + } +} diff --git a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift index b9af2b638290..f0fbe30d0fc3 100644 --- a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift +++ b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift @@ -10,31 +10,44 @@ import XCTest @testable import Brave final class PageDataTests: XCTestCase { + @MainActor private lazy var ruleStore: WKContentRuleListStore = { + let testBundle = Bundle.module + let bundleURL = testBundle.bundleURL + return WKContentRuleListStore(url: bundleURL)! + }() + @MainActor func testBasicExample() throws { // Given // Page data with empty ad-block stats let mainFrameURL = URL(string: "http://example.com")! let subFrameURL = URL(string: "http://example.com/1p/subframe")! let upgradedMainFrameURL = URL(string: "https://example.com")! - var pageData = PageData( - mainFrameURL: mainFrameURL, - groupsManager: AdBlockGroupsManager( - standardManager: AdBlockEngineManager( - engineType: .standard, - cacheFolderName: "test_standard" - ), - aggressiveManager: AdBlockEngineManager( - engineType: .standard, - cacheFolderName: "test_aggressive" - ) - ) + let sourceProvider = TestSourceProvider() + let groupsManager = AdBlockGroupsManager( + standardManager: AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ), + aggressiveManager: AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_aggressive" + ), + contentBlockerManager: makeContentBlockingManager(), + sourceProvider: sourceProvider ) + + var pageData = PageData(mainFrameURL: mainFrameURL, groupsManager: groupsManager) let expectation = expectation(description: "") Task { @MainActor in // When // We get the script types for the main frame let domain = pageData.domain(persistent: false) + // sourceProvider.set(source: sourceProvider.fileInfos[0].filterListInfo.source, enabled: true) + // sourceProvider.set(source: sourceProvider.fileInfos[1].filterListInfo.source, enabled: true) + // await groupsManager.updated(fileInfo: sourceProvider.fileInfos[0], engineType: .standard) + // await groupsManager.compileEnginesIfNeeded() + let mainFrameRequestTypes = await pageData.makeUserScriptTypes( domain: domain, isDeAmpEnabled: false @@ -115,4 +128,8 @@ final class PageDataTests: XCTestCase { waitForExpectations(timeout: 10) } + + @MainActor private func makeContentBlockingManager() -> ContentBlockerManager { + return ContentBlockerManager(ruleStore: ruleStore) + } } diff --git a/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift index d0203c7681b0..22ff85f37bfa 100644 --- a/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift +++ b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift @@ -57,7 +57,7 @@ final class GroupedAdBlockEngineTests: XCTestCase { ) } - func testEngineMemoryManagment() throws { + @MainActor func testEngineMemoryManagment() throws { AdblockEngine.setDomainResolver() var engine: AdblockEngine? = AdblockEngine() weak var weakEngine: AdblockEngine? = engine @@ -116,14 +116,10 @@ final class GroupedAdBlockEngineTests: XCTestCase { do { let engine = try GroupedAdBlockEngine.compile(group: group, type: .standard) try await engine.useResources(from: resourcesInfo) - let url = URL(string: "https://stackoverflow.com")! - - let domain = await MainActor.run { - return Domain.getOrCreate(forUrl: url, persistent: false) - } + let frameURL = URL(string: "https://stackoverflow.com")! let sameDomainTypes = try await engine.makeEngineScriptTypes( - frameURL: url, isMainFrame: true, domain: domain, isDeAmpEnabled: false, index: 0 + frameURL: frameURL, isMainFrame: true, isDeAmpEnabled: false, index: 0 ) // We should have no scripts injected @@ -132,7 +128,7 @@ final class GroupedAdBlockEngineTests: XCTestCase { if await engine.group.infos.contains(textFilterListInfo) { // This engine file contains some scriplet rules so we can test this part is working let crossDomainTypes = try await engine.makeEngineScriptTypes( - frameURL: URL(string: "https://reddit.com")!, isMainFrame: true, domain: domain, + frameURL: URL(string: "https://reddit.com")!, isMainFrame: true, isDeAmpEnabled: false, index: 0 ) From 807657375a44012abdfef330a95dc95eb669fec6 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Fri, 29 Mar 2024 01:00:24 +0100 Subject: [PATCH 06/10] Fix issues --- .../FilterLists/FilterListsView.swift | 17 ++++ .../AdBlock/AdBlockEngineManager.swift | 90 +++++++++++-------- .../AdBlock/AdBlockGroupsManager.swift | 87 +++++++++++++----- .../ContentBlockerManager.swift | 45 ++++++++-- .../FilterListCustomURLDownloader.swift | 7 +- .../FilterListResourceDownloader.swift | 2 +- .../AdBlockEngineManagerTests.swift | 7 +- .../AdBlockGroupsManagerTests.swift | 9 +- .../Tests/ClientTests/PageDataTests.swift | 4 - 9 files changed, 187 insertions(+), 81 deletions(-) diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift index 3d577fe98df6..d48238565208 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift @@ -71,6 +71,23 @@ struct FilterListsView: View { } @ViewBuilder private var filterListView: some View { + #if DEBUG + let allEnabled = Binding { + filterListStorage.filterLists.allSatisfy({ $0.isEnabled }) + } set: { isEnabled in + filterListStorage.filterLists.enumerated().forEach { index, filterList in + let isEnabled = filterList.entry.hidden ? filterList.entry.defaultEnabled : isEnabled + filterListStorage.filterLists[index].isEnabled = isEnabled + } + } + + Toggle(isOn: allEnabled) { + VStack(alignment: .leading) { + Text("All").foregroundColor(Color(.bravePrimary)) + } + } + #endif + ForEach($filterListStorage.filterLists) { $filterList in if !filterList.isHidden { Toggle(isOn: $filterList.isEnabled) { diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index 6cbe353c641a..c6cba6ef6987 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -76,7 +76,10 @@ import os for enabledSources: [GroupedAdBlockEngine.Source] ) -> [FileInfo] { return enabledSources.compactMap { source in - return availableFiles.first(where: { $0.filterListInfo.source == source }) + return availableFiles.first(where: { + FileManager.default.fileExists(atPath: $0.localFileURL.path) + && $0.filterListInfo.source == source + }) } } @@ -98,9 +101,13 @@ import os } /// Checks to see if we need to compile or recompile the engine based on the available info - func checkNeedsCompile(for enabledSources: [GroupedAdBlockEngine.Source]) -> Bool { - let compilableInfos = compilableFiles(for: enabledSources).map({ $0.filterListInfo }) - guard !compilableInfos.isEmpty else { return false } + func checkNeedsCompile(for fileInfos: [AdBlockEngineManager.FileInfo]) -> Bool { + if let pendingGroup = pendingGroup { + return pendingGroup.infos != fileInfos.map({ $0.filterListInfo }) + } + + let compilableInfos = fileInfos.map({ $0.filterListInfo }) + guard !compilableInfos.isEmpty else { return engine?.group.infos.isEmpty == false } guard let engine = engine else { return true } return compilableInfos != engine.group.infos } @@ -145,22 +152,19 @@ import os /// Especially needed during launch when we have a bunch of downloads coming at the same time. func compileDelayedIfNeeded( for enabledSources: [GroupedAdBlockEngine.Source], - resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, - priority: TaskPriority + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? ) { - guard self.checkNeedsCompile(for: enabledSources) else { return } - // Cancel the previous task delayTask?.cancel() // Restart the task delayTask = Task { let hasAllInfo = checkHasAllInfo(for: enabledSources) - try await Task.sleep(seconds: hasAllInfo ? 1 : 60) + try await Task.sleep(seconds: hasAllInfo ? 5 : 60) + await compileAvailableIfNeeded( for: enabledSources, - resourcesInfo: resourcesInfo, - priority: priority + resourcesInfo: resourcesInfo ) } } @@ -168,15 +172,13 @@ import os /// This will compile available data right away if it is needed and cancel any delayedTasks func compileImmediatelyIfNeeded( for enabledSources: [GroupedAdBlockEngine.Source], - resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, - priority: TaskPriority + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? ) async { delayTask?.cancel() await self.compileAvailableIfNeeded( for: enabledSources, - resourcesInfo: resourcesInfo, - priority: priority + resourcesInfo: resourcesInfo ) } @@ -192,29 +194,37 @@ import os } func needsCompile(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) -> Bool { - guard let info = engine?.group.infos.first(where: { $0.source == filterListInfo.source }) else { + var infos = availableFiles.map(\.filterListInfo) + if let engineInfos = engine?.group.infos { + // This is an optimization because during launch we don't have file infos. + // But we will have an engine loaded from cache. + // So we also check the engine infos as well + infos.append(contentsOf: engineInfos) + } + + guard let info = infos.first(where: { $0.source == filterListInfo.source }) else { return true } + return filterListInfo.version < info.version } - /// Checks to see if we need to compile or recompile the engine based on the available info - private func checkHasAllInfo(for sources: [GroupedAdBlockEngine.Source]) -> Bool { - return Set(sources) == Set(availableFiles.map(\.filterListInfo.source)) + func checkHasAllInfo(for sources: [GroupedAdBlockEngine.Source]) -> Bool { + let availableSources = compilableFiles(for: sources).map({ $0.filterListInfo.source }) + return sources.allSatisfy({ availableSources.contains($0) }) } /// This will compile available data right away if it is needed private func compileAvailableIfNeeded( for enabledSources: [GroupedAdBlockEngine.Source], - resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, - priority: TaskPriority + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? ) async { do { - guard self.checkNeedsCompile(for: enabledSources) else { return } + let compilableFiles = compilableFiles(for: enabledSources) + guard self.checkNeedsCompile(for: compilableFiles) else { return } try await compileAvailable( - for: enabledSources, - resourcesInfo: resourcesInfo, - priority: priority + for: compilableFiles, + resourcesInfo: resourcesInfo ) } catch { ContentBlockerManager.log.error( @@ -225,16 +235,23 @@ import os /// Compile an engine from all available data private func compileAvailable( - for enabledSources: [GroupedAdBlockEngine.Source], - resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?, - priority: TaskPriority + for files: [AdBlockEngineManager.FileInfo], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? ) async throws { + let infosString = files.map({ " \($0.filterListInfo.debugDescription)" }) + .joined(separator: "\n") + let count = files.count + + ContentBlockerManager.log.debug( + "Compiling `\(self.cacheFolderName)` engine from \(count) sources:\n\(infosString)" + ) + let engineType = self.engineType - let group = try combineRules(for: enabledSources) + let group = try combineRules(for: files) self.pendingGroup = group // 2. Compile the engine - let groupedEngine = try await Task.detached(priority: priority) { + let groupedEngine = try await Task.detached(priority: .high) { let engine = try GroupedAdBlockEngine.compile(group: group, type: engineType) if let resourcesInfo { @@ -264,17 +281,14 @@ import os /// Take all the filter lists and combine them into one then save them into a cache folder. private func combineRules( - for enabledSources: [GroupedAdBlockEngine.Source] + for compilableFiles: [AdBlockEngineManager.FileInfo] ) throws -> GroupedAdBlockEngine.FilterListGroup { - // 1. Grab all the needed files - let compilableFiles = compilableFiles(for: enabledSources) - - // 2. Create a file url + // 1. Create a file url let cachedFolder = try getOrCreateCacheFolder() let fileURL = cachedFolder.appendingPathComponent("list.txt", conformingTo: .text) var compiledInfos: [GroupedAdBlockEngine.FilterListInfo] = [] var unifiedRules = "" - // 3. Join all the rules together + // 2. Join all the rules together compilableFiles.forEach { fileInfo in do { let fileContents = try String(contentsOf: fileInfo.localFileURL) @@ -282,12 +296,12 @@ import os unifiedRules = [unifiedRules, fileContents].joined(separator: "\n") } catch { ContentBlockerManager.log.error( - "Could not load rules for `\(fileInfo.filterListInfo.debugDescription)`: \(error)" + "Could not load rules for \(fileInfo.filterListInfo.debugDescription): \(error)" ) } } - // 4. Save the files into storage + // 3. Save the files into storage if FileManager.default.fileExists(atPath: fileURL.path) { try FileManager.default.removeItem(at: fileURL) } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 1e7faf5993c4..0a48e310a68d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -96,8 +96,7 @@ import os await manager.compileImmediatelyIfNeeded( for: sourceProvider.enabledSources, - resourcesInfo: self.resourcesInfo, - priority: .high + resourcesInfo: self.resourcesInfo ) } } @@ -118,27 +117,75 @@ import os updateIfNeeded(resourcesInfo: resourcesInfo) } - /// Handle updated filter list info - func updated( - fileInfo: AdBlockEngineManager.FileInfo, + func update( + fileInfos: [AdBlockEngineManager.FileInfo], engineType: GroupedAdBlockEngine.EngineType - ) async { + ) { let manager = getManager(for: engineType) let enabledSources = sourceProvider.enabledSources // Compile content blockers if this filter list is enabled - if enabledSources.contains(fileInfo.filterListInfo.source) { - await ensureContentBlockers(for: fileInfo, engineType: engineType) + for fileInfo in fileInfos { + if enabledSources.contains(fileInfo.filterListInfo.source) { + Task { + await ensureContentBlockers(for: fileInfo, engineType: engineType) + } + } + + manager.add(fileInfo: fileInfo) + } + + let sources = fileInfos.map({ $0.filterListInfo.source }) + + if manager.checkHasAllInfo(for: sources) { + Task { + await manager.compileImmediatelyIfNeeded( + for: enabledSources, + resourcesInfo: self.resourcesInfo + ) + } + } else { + manager.compileDelayedIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo + ) + } + } + + /// Handle updated filter list info + func update( + fileInfo: AdBlockEngineManager.FileInfo, + engineType: GroupedAdBlockEngine.EngineType + ) { + update(fileInfos: [fileInfo], engineType: engineType) + } + + func removeFileInfos( + for sources: [GroupedAdBlockEngine.Source], + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + for source in sources { + manager.removeInfo(for: source) } - // Always update the info on the manager - manager.add(fileInfo: fileInfo) manager.compileDelayedIfNeeded( - for: enabledSources, - resourcesInfo: resourcesInfo, - priority: .background + for: sourceProvider.enabledSources, + resourcesInfo: resourcesInfo ) + } + func removeFileInfo( + for source: GroupedAdBlockEngine.Source, + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + manager.removeInfo(for: source) + + manager.compileDelayedIfNeeded( + for: sourceProvider.enabledSources, + resourcesInfo: resourcesInfo + ) } /// Ensure all engines and content blockers are compiled @@ -148,8 +195,7 @@ import os let manager = self.getManager(for: engineType) await manager.compileImmediatelyIfNeeded( for: enabledSources, - resourcesInfo: self.resourcesInfo, - priority: .high + resourcesInfo: self.resourcesInfo ) self.ensureContentBlockers(for: enabledSources, engineType: engineType) @@ -354,12 +400,13 @@ extension AdBlockEngineManager.FileInfo { return nil } - let filterListInfo = GroupedAdBlockEngine.FilterListInfo( - source: source, - version: version + self.init( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: source, + version: version + ), + localFileURL: localFileURL ) - - self.init(filterListInfo: filterListInfo, localFileURL: localFileURL) } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift index 416c7fd79e6e..04033a2a326d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift @@ -70,6 +70,20 @@ actor ContentBlockerManager { } } + enum CompileResult { + case empty + case success(WKContentRuleList) + case failure(Error) + + func get() throws -> WKContentRuleList? { + switch self { + case .empty: return nil + case .success(let ruleList): return ruleList + case .failure(let error): throw error + } + } + } + /// An object representing the type of block list public enum BlocklistType: Hashable, CustomDebugStringConvertible { fileprivate static let genericPrifix = "stored-type" @@ -139,7 +153,7 @@ actor ContentBlockerManager { /// The store in which these rule lists should be compiled let ruleStore: WKContentRuleListStore /// We cached the rule lists so that we can return them quicker if we need to - private var cachedRuleLists: [String: Result] + private var cachedRuleLists: [String: CompileResult] /// A list of etld+1s that are always aggressive /// TODO: @JS Replace this with the 1st party ad-block list let alwaysAggressiveETLDs: Set = ["youtube.com"] @@ -210,7 +224,7 @@ actor ContentBlockerManager { try await compile(encodedContentRuleList: result.rulesJSON, for: blocklistType, modes: modes) } catch { ContentBlockerManager.log.error( - "Failed to compile rule list for \(blocklistType.debugDescription)" + "Failed to compile rule list for `\(blocklistType.debugDescription)`" ) } } @@ -223,13 +237,25 @@ actor ContentBlockerManager { ) async throws { guard !modes.isEmpty else { return } var foundError: Error? + let ruleList = try decode(encodedContentRuleList: encodedContentRuleList) + + guard !ruleList.isEmpty else { + for mode in modes { + self.cachedRuleLists[type.makeIdentifier(for: mode)] = .empty + } + + ContentBlockerManager.log.debug( + "Empty filter set for `\(type.debugDescription)`" + ) + return + } for mode in modes { let identifier = type.makeIdentifier(for: mode) do { let moddedRuleList = try self.modify( - encodedContentRuleList: encodedContentRuleList, + ruleList: ruleList, for: mode ) let ruleList = try await compile( @@ -237,6 +263,7 @@ actor ContentBlockerManager { for: type, mode: mode ) + self.cachedRuleLists[identifier] = .success(ruleList) Self.log.debug("Compiled rule list for `\(identifier)`") } catch { @@ -253,16 +280,18 @@ actor ContentBlockerManager { } } - private func modify(encodedContentRuleList: String, for mode: BlockingMode) throws -> String? { + private func modify( + ruleList: [[String: Any?]], + for mode: BlockingMode + ) throws -> String? { switch mode { case .aggressive, .general: // Aggressive mode and general mode has no modification to the rules - return nil + let modifiedData = try JSONSerialization.data(withJSONObject: ruleList) + return String(bytes: modifiedData, encoding: .utf8) case .standard: - // Add the ignore first party rule to make it standard - var ruleList = try decode(encodedContentRuleList: encodedContentRuleList) - + var ruleList = ruleList // We need to make sure we are not going over the limit // So we make space for the added rule if ruleList.count >= (Self.maxContentBlockerSize) { diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift index c9cd286fafdd..02ce3094e50d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift @@ -76,7 +76,7 @@ actor FilterListCustomURLDownloader: ObservableObject { localFileURL: downloadResult.fileURL ) - await AdBlockGroupsManager.shared.updated( + await AdBlockGroupsManager.shared.update( fileInfo: fileInfo, engineType: .aggressive ) @@ -142,6 +142,11 @@ actor FilterListCustomURLDownloader: ObservableObject { let resource = await filterListCustomURL.setting.resource fetchTasks[resource]?.cancel() fetchTasks.removeValue(forKey: resource) + + await AdBlockGroupsManager.shared.removeFileInfo( + for: filterListCustomURL.setting.engineSource, + engineType: .aggressive + ) } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index 5daf32a3f5f8..b6b3ca8fb81c 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -94,7 +94,7 @@ public actor FilterListResourceDownloader { return } - await AdBlockGroupsManager.shared.updated( + await AdBlockGroupsManager.shared.update( fileInfo: fileInfo, engineType: engineType ) diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift index 470298bb7848..e4a2002a3fb9 100644 --- a/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift +++ b/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift @@ -49,7 +49,7 @@ final class AdBlockEngineManagerTests: XCTestCase { // Then // Needs compile returns true and there is no engine - var needsCompile = await engineManager.checkNeedsCompile(for: sources) + var needsCompile = await engineManager.checkNeedsCompile(for: fileInfos) var engine = await engineManager.engine XCTAssertTrue(needsCompile) XCTAssertNil(engine) @@ -68,13 +68,12 @@ final class AdBlockEngineManagerTests: XCTestCase { // We compile engine await engineManager.compileImmediatelyIfNeeded( for: sources, - resourcesInfo: resourcesInfo, - priority: .high + resourcesInfo: resourcesInfo ) // Then // Needs compile returns false and engine is correctly created - needsCompile = await engineManager.checkNeedsCompile(for: sources) + needsCompile = await engineManager.checkNeedsCompile(for: fileInfos) engine = await engineManager.engine let compiledResources = await engine?.resourcesInfo let group = await engine?.group diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift index 27d358d1473c..a90e673586c1 100644 --- a/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift +++ b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift @@ -71,9 +71,8 @@ final class AdBlockGroupsManagerTests: XCTestCase { enabled: true ) await groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) - await groupsManager.updated(fileInfo: fileInfos[1], engineType: .standard) - await groupsManager.updated(fileInfo: fileInfos[0], engineType: .aggressive) - await groupsManager.updated(fileInfo: fileInfos[1], engineType: .aggressive) + await groupsManager.update(fileInfo: fileInfos[1], engineType: .standard) + await groupsManager.update(fileInfos: fileInfos, engineType: .aggressive) await groupsManager.compileEnginesIfNeeded() // Then @@ -154,7 +153,7 @@ final class AdBlockGroupsManagerTests: XCTestCase { enabled: true ) groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) - await groupsManager.updated(fileInfo: fileInfo, engineType: .standard) + groupsManager.update(fileInfo: fileInfo, engineType: .standard) await groupsManager.compileEnginesIfNeeded() let mainFrameURL = URL(string: "https://dev-pages.bravesoftware.com")! @@ -218,7 +217,7 @@ final class AdBlockGroupsManagerTests: XCTestCase { // When // Adding an aggressive filter list - await groupsManager.updated(fileInfo: fileInfo, engineType: .aggressive) + groupsManager.update(fileInfo: fileInfo, engineType: .aggressive) await groupsManager.compileEnginesIfNeeded() // Then diff --git a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift index f0fbe30d0fc3..88f75be0f6a3 100644 --- a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift +++ b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift @@ -43,10 +43,6 @@ final class PageDataTests: XCTestCase { // When // We get the script types for the main frame let domain = pageData.domain(persistent: false) - // sourceProvider.set(source: sourceProvider.fileInfos[0].filterListInfo.source, enabled: true) - // sourceProvider.set(source: sourceProvider.fileInfos[1].filterListInfo.source, enabled: true) - // await groupsManager.updated(fileInfo: sourceProvider.fileInfos[0], engineType: .standard) - // await groupsManager.compileEnginesIfNeeded() let mainFrameRequestTypes = await pageData.makeUserScriptTypes( domain: domain, From 66887e71ec6f1b7c2212bc1a505149435eaefd0c Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Sat, 30 Mar 2024 14:23:01 +0100 Subject: [PATCH 07/10] Fix tests Remove warning --- .../AdBlock/AdBlockGroupsManager.swift | 19 +++--------- .../AdBlockGroupsManagerTests.swift | 30 +++++++++---------- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 0a48e310a68d..1afcbd75d8e6 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -135,21 +135,10 @@ import os manager.add(fileInfo: fileInfo) } - let sources = fileInfos.map({ $0.filterListInfo.source }) - - if manager.checkHasAllInfo(for: sources) { - Task { - await manager.compileImmediatelyIfNeeded( - for: enabledSources, - resourcesInfo: self.resourcesInfo - ) - } - } else { - manager.compileDelayedIfNeeded( - for: enabledSources, - resourcesInfo: resourcesInfo - ) - } + manager.compileDelayedIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo + ) } /// Handle updated filter list info diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift index a90e673586c1..aea730f0c445 100644 --- a/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift +++ b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift @@ -23,7 +23,7 @@ final class AdBlockGroupsManagerTests: XCTestCase { }() /// Testing engine compilations and filter list managment - func testCompilation() async throws { + @MainActor func testCompilation() async throws { AdblockEngine.setDomainResolver() // Given @@ -44,16 +44,16 @@ final class AdBlockGroupsManagerTests: XCTestCase { localFileURL: Self.sampleFilterListURL ), ] - let sourceProvider = await TestSourceProvider(fileInfos: fileInfos) - let standardManager = await AdBlockEngineManager( + let sourceProvider = TestSourceProvider(fileInfos: fileInfos) + let standardManager = AdBlockEngineManager( engineType: .standard, cacheFolderName: "test_standard" ) - let aggressiveManager = await AdBlockEngineManager( + let aggressiveManager = AdBlockEngineManager( engineType: .aggressive, cacheFolderName: "test_aggressive" ) - let groupsManager = await AdBlockGroupsManager( + let groupsManager = AdBlockGroupsManager( standardManager: standardManager, aggressiveManager: aggressiveManager, contentBlockerManager: makeContentBlockingManager(), @@ -66,19 +66,19 @@ final class AdBlockGroupsManagerTests: XCTestCase { // When // Adding file info and enabling sources for only one engine - await sourceProvider.set( + sourceProvider.set( source: fileInfos[0].filterListInfo.source, enabled: true ) - await groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) - await groupsManager.update(fileInfo: fileInfos[1], engineType: .standard) - await groupsManager.update(fileInfos: fileInfos, engineType: .aggressive) + groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + groupsManager.update(fileInfo: fileInfos[1], engineType: .standard) + groupsManager.update(fileInfos: fileInfos, engineType: .aggressive) await groupsManager.compileEnginesIfNeeded() // Then // Only one engine is created - var standardEngine = await standardManager.engine - var aggressiveEngine = await aggressiveManager.engine + var standardEngine = standardManager.engine + var aggressiveEngine = aggressiveManager.engine var standardGroup = await standardEngine?.group var aggressiveGroup = await aggressiveEngine?.group var standardResources = await standardEngine?.resourcesInfo @@ -90,17 +90,17 @@ final class AdBlockGroupsManagerTests: XCTestCase { // When // We enable sources and recompile the engine - await sourceProvider.set( + sourceProvider.set( source: fileInfos[1].filterListInfo.source, enabled: true ) - await groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) await groupsManager.compileEnginesIfNeeded() // Then // All engines are created - standardEngine = await standardManager.engine - aggressiveEngine = await aggressiveManager.engine + standardEngine = standardManager.engine + aggressiveEngine = aggressiveManager.engine standardGroup = await standardEngine?.group aggressiveGroup = await aggressiveEngine?.group standardResources = await standardEngine?.resourcesInfo From 4baeedce1a9217fee7b687c58298931dc145caf6 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Sat, 30 Mar 2024 15:46:41 +0100 Subject: [PATCH 08/10] Logging improvements --- .../AdBlock/AdBlockEngineManager.swift | 24 +++++++++---------- .../AdBlock/GroupedAdBlockEngine.swift | 21 +++++++++++----- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index c6cba6ef6987..60528014daf1 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -238,18 +238,17 @@ import os for files: [AdBlockEngineManager.FileInfo], resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? ) async throws { - let infosString = files.map({ " \($0.filterListInfo.debugDescription)" }) - .joined(separator: "\n") - let count = files.count - - ContentBlockerManager.log.debug( - "Compiling `\(self.cacheFolderName)` engine from \(count) sources:\n\(infosString)" - ) - let engineType = self.engineType let group = try combineRules(for: files) self.pendingGroup = group + ContentBlockerManager.log.debug( + """ + Compiling `\(self.cacheFolderName)` engine from \(group.infos.count) sources: + \(group.debugDescription)" + """ + ) + // 2. Compile the engine let groupedEngine = try await Task.detached(priority: .high) { let engine = try GroupedAdBlockEngine.compile(group: group, type: engineType) @@ -270,11 +269,12 @@ import os } private func set(engine: GroupedAdBlockEngine) { - let infosString = engine.group.infos.map({ " \($0.debugDescription)" }).joined(separator: "\n") - let fileTypeString = engine.group.fileType.debugDescription - let count = engine.group.infos.count + let group = engine.group ContentBlockerManager.log.debug( - "Set `\(self.cacheFolderName)` (\(fileTypeString)) engine from \(count) sources:\n\(infosString)" + """ + Set `\(self.cacheFolderName)` (\(group.fileType.debugDescription)) engine from \(group.infos.count) sources: + \(group.debugDescription)" + """ ) self.engine = engine } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift index 21ad0951d8bc..fca4404dcb4c 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift @@ -7,7 +7,7 @@ import BraveCore import Data import Foundation import Preferences -import os +import os.log /// An object that wraps around an `AdblockEngine` and caches some results /// and ensures information is always returned on the correct thread on the engine. @@ -18,8 +18,8 @@ public actor GroupedAdBlockEngine { public var debugDescription: String { switch self { - case .filterList(let componentId, _): return "filterList(\(componentId))" - case .filterListURL(let uuid): return "filterListURL(\(uuid))" + case .filterList(let componentId, _): return componentId + case .filterListURL(let uuid): return uuid } } } @@ -35,7 +35,7 @@ public actor GroupedAdBlockEngine { } } - public enum EngineType: Hashable, CaseIterable { + public enum EngineType: Hashable, CaseIterable, CustomDebugStringConvertible { case standard case aggressive @@ -45,6 +45,13 @@ public actor GroupedAdBlockEngine { case .aggressive: return true } } + + public var debugDescription: String { + switch self { + case .aggressive: return "aggressive" + case .standard: return "standard" + } + } } public struct FilterListInfo: Codable, Hashable, Equatable, CustomDebugStringConvertible { @@ -62,7 +69,9 @@ public actor GroupedAdBlockEngine { let fileType: GroupedAdBlockEngine.FileType public var debugDescription: String { - return infos.map({ $0.debugDescription }).joined(separator: ", ") + return infos.enumerated() + .map({ " #\($0) \($1.debugDescription)" }) + .joined(separator: "\n") } } @@ -207,7 +216,7 @@ public actor GroupedAdBlockEngine { let state = Self.signpost.beginInterval( "compileEngine", id: signpostID, - "\(group.debugDescription)" + "\(type.debugDescription) (\(group.fileType.debugDescription)): \(group.debugDescription)" ) do { From 55df8c2105e962309c204b7128611da80dfadf05 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Wed, 3 Apr 2024 20:36:32 +0200 Subject: [PATCH 09/10] Fix for review --- .../Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift index 60528014daf1..7f345e33ff28 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -13,7 +13,7 @@ import os @MainActor class AdBlockEngineManager { /// The directory to which we should store the unified list into private static var cacheFolderDirectory: FileManager.SearchPathDirectory { - return FileManager.SearchPathDirectory.applicationSupportDirectory + return FileManager.SearchPathDirectory.cachesDirectory } public struct FileInfo: Hashable, Equatable { From 776152c78c0be5df3bd956b06599b1339bf01e87 Mon Sep 17 00:00:00 2001 From: Jacob Sikorski Date: Thu, 4 Apr 2024 02:21:18 +0200 Subject: [PATCH 10/10] Don't migrate non-default filter lists --- .../Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift index 1afcbd75d8e6..f61be437d544 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -90,6 +90,10 @@ import os let manager = getManager(for: engineType) if await !manager.loadFromCache(resourcesInfo: self.resourcesInfo) { + // This migration will add ~24s on an iPhone 8 (~8s on an iPhone 14) + // Even though its a one time thing, let's skip it. + // We never waited for the aggressive engines to be ready before anyways + guard engineType == .standard else { return } for fileInfo in sourceProvider.legacyCacheFiles(for: engineType) { manager.add(fileInfo: fileInfo) }