diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift index 3e07116d64c0..79c7fd95e4db 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BVC+WKNavigationDelegate.swift @@ -436,11 +436,11 @@ extension BrowserViewController: WKNavigationDelegate { { let domain = Domain.getOrCreate(forUrl: requestURL, persistent: !isPrivateBrowsing) - let shouldBlock = await AdBlockStats.shared.shouldBlock( + let shouldBlock = await AdBlockGroupsManager.shared.shouldBlock( requestURL: requestURL, sourceURL: requestURL, resourceType: .document, - isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + domain: domain ) if shouldBlock, let url = requestURL.encodeEmbeddedInternalURL(for: .blocked) { diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift index 64a036ed172e..feff1dec99c3 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/BrowserViewController/BrowserViewController.swift @@ -468,7 +468,7 @@ public class BrowserViewController: UIViewController { ScriptFactory.shared.clearCaches() Task { - await AdBlockStats.shared.didReceiveMemoryWarning() + await AdBlockGroupsManager.shared.didReceiveMemoryWarning() } for tab in tabManager.tabsForCurrentMode where tab.id != tabManager.selectedTab?.id { diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift index 8a8dcd5d9abb..f783d89e2adc 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/Helpers/LaunchHelper.swift @@ -48,11 +48,12 @@ public actor LaunchHelper { // Load cached data // This is done first because compileResources need their results - async let filterListCache: Void = FilterListResourceDownloader.shared - .loadFilterListSettingsAndCachedData() + await FilterListStorage.shared.loadFilterListSettings() + await AdBlockGroupsManager.shared.loadResourcesFromCache() + async let loadEngines: Void = AdBlockGroupsManager.shared.loadEnginesFromCache() async let adblockResourceCache: Void = AdblockResourceDownloader.shared .loadCachedAndBundledDataIfNeeded(allowedModes: launchBlockModes) - _ = await (filterListCache, adblockResourceCache) + _ = await (loadEngines, adblockResourceCache) Self.signpost.emitEvent("loadedCachedData", id: signpostID, "Loaded cached data") ContentBlockerManager.log.debug("Loaded blocking launch data") @@ -101,22 +102,10 @@ public actor LaunchHelper { let signpostID = Self.signpost.makeSignpostID() let state = Self.signpost.beginInterval("nonBlockingLaunchTask", id: signpostID) await FilterListResourceDownloader.shared.start(with: adBlockService) - Self.signpost.emitEvent( - "FilterListResourceDownloader.shared.start", - id: signpostID, - "Started filter list downloader" - ) await AdblockResourceDownloader.shared.loadCachedAndBundledDataIfNeeded( allowedModes: Set(remainingModes) ) - Self.signpost.emitEvent( - "loadCachedAndBundledDataIfNeeded", - id: signpostID, - "Reloaded data for remaining modes" - ) await AdblockResourceDownloader.shared.startFetching() - Self.signpost.emitEvent("startFetching", id: signpostID, "Started fetching ad-block data") - /// Cleanup rule lists so we don't have dead rule lists let validBlocklistTypes = await self.getAllValidBlocklistTypes() await ContentBlockerManager.shared.cleaupInvalidRuleLists(validTypes: validBlocklistTypes) @@ -153,9 +142,7 @@ extension FilterListStorage { // If we don't have filter lists yet loaded, use the settings return Set( allFilterListSettings.compactMap { setting -> ContentBlockerManager.BlocklistType? in - guard let componentId = setting.componentId else { return nil } - return .filterList( - componentId: componentId, + return setting.engineSource?.blocklistType( isAlwaysAggressive: setting.isAlwaysAggressive ) } @@ -163,10 +150,9 @@ extension FilterListStorage { } else { // If we do have filter lists yet loaded, use them as they are always the most up to date and accurate return Set( - filterLists.map { filterList in - return .filterList( - componentId: filterList.entry.componentId, - isAlwaysAggressive: filterList.isAlwaysAggressive + filterLists.compactMap { filterList in + return filterList.engineSource.blocklistType( + isAlwaysAggressive: filterList.engineType.isAlwaysAggressive ) } ) diff --git a/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift b/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift index 1761ac07491e..004b3d9af1a7 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Browser/PageData.swift @@ -22,11 +22,11 @@ struct PageData { /// These are loaded dyncamically as the user scrolls through the page private(set) var allSubframeURLs: Set = [] /// The stats class to get the engine data from - private var adBlockStats: AdBlockStats + private var groupsManager: AdBlockGroupsManager - init(mainFrameURL: URL, adBlockStats: AdBlockStats = AdBlockStats.shared) { + init(mainFrameURL: URL, groupsManager: AdBlockGroupsManager = AdBlockGroupsManager.shared) { self.mainFrameURL = mainFrameURL - self.adBlockStats = adBlockStats + self.groupsManager = groupsManager } /// This method builds all the user scripts that should be included for this page @@ -120,7 +120,7 @@ struct PageData { domain: Domain, isDeAmpEnabled: Bool ) async -> Set { - return await adBlockStats.makeEngineScriptTypes( + return await groupsManager.makeEngineScriptTypes( frameURL: mainFrameURL, isMainFrame: true, isDeAmpEnabled: isDeAmpEnabled, @@ -130,7 +130,7 @@ struct PageData { func makeAllEngineScripts(for domain: Domain, isDeAmpEnabled: Bool) async -> Set { // Add engine scripts for the main frame - async let engineScripts = adBlockStats.makeEngineScriptTypes( + async let engineScripts = groupsManager.makeEngineScriptTypes( frameURL: mainFrameURL, isMainFrame: true, isDeAmpEnabled: isDeAmpEnabled, @@ -139,7 +139,7 @@ struct PageData { // Add engine scripts for all of the known sub-frames async let additionalScriptTypes = allSubframeURLs.asyncConcurrentCompactMap({ frameURL in - return await self.adBlockStats.makeEngineScriptTypes( + return await self.groupsManager.makeEngineScriptTypes( frameURL: frameURL, isMainFrame: false, isDeAmpEnabled: isDeAmpEnabled, diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift index 35ce2f878e57..210a30bd0f0f 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/AdvancedShieldSettings.swift @@ -28,17 +28,25 @@ import os @Published var cookieConsentBlocking: Bool { didSet { FilterListStorage.shared.ensureFilterList( - for: FilterList.cookieConsentNoticesComponentID, + for: AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID, isEnabled: cookieConsentBlocking ) + + Task { + await AdBlockGroupsManager.shared.compileEnginesIfNeeded() + } } } @Published var blockMobileAnnoyances: Bool { didSet { FilterListStorage.shared.ensureFilterList( - for: FilterList.mobileAnnoyancesComponentID, + for: AdblockFilterListCatalogEntry.mobileAnnoyancesComponentID, isEnabled: blockMobileAnnoyances ) + + Task { + await AdBlockGroupsManager.shared.compileEnginesIfNeeded() + } } } @Published var isP3AEnabled: Bool { @@ -101,11 +109,11 @@ import os self.isDebounceEnabled = debounceService?.isEnabled ?? false cookieConsentBlocking = FilterListStorage.shared.isEnabled( - for: FilterList.cookieConsentNoticesComponentID + for: AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID ) blockMobileAnnoyances = FilterListStorage.shared.isEnabled( - for: FilterList.mobileAnnoyancesComponentID + for: AdblockFilterListCatalogEntry.mobileAnnoyancesComponentID ) var clearableSettings = [ @@ -232,11 +240,11 @@ import os .sink { filterLists in for filterList in filterLists { switch filterList.entry.componentId { - case FilterList.cookieConsentNoticesComponentID: + case AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID: if filterList.isEnabled != self.cookieConsentBlocking { self.cookieConsentBlocking = filterList.isEnabled } - case FilterList.mobileAnnoyancesComponentID: + case AdblockFilterListCatalogEntry.mobileAnnoyancesComponentID: if filterList.isEnabled != self.blockMobileAnnoyances { self.blockMobileAnnoyances = filterList.isEnabled } diff --git a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift index 023321469e6c..d48238565208 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift @@ -16,9 +16,6 @@ struct FilterListsView: View { @ObservedObject private var customFilterListStorage = CustomFilterListStorage.shared @Environment(\.editMode) private var editMode @State private var showingAddSheet = false - @State private var expectedEnabledSources: Set = Set( - AdBlockStats.shared.enabledSources - ) private let dateFormatter = RelativeDateTimeFormatter() var body: some View { @@ -67,14 +64,30 @@ struct FilterListsView: View { ) } .onDisappear { - Task.detached { - await AdBlockStats.shared.removeDisabledEngines() - await AdBlockStats.shared.ensureEnabledEngines() + Task { + await AdBlockGroupsManager.shared.compileEnginesIfNeeded() } } } @ViewBuilder private var filterListView: some View { + #if DEBUG + let allEnabled = Binding { + filterListStorage.filterLists.allSatisfy({ $0.isEnabled }) + } set: { isEnabled in + filterListStorage.filterLists.enumerated().forEach { index, filterList in + let isEnabled = filterList.entry.hidden ? filterList.entry.defaultEnabled : isEnabled + filterListStorage.filterLists[index].isEnabled = isEnabled + } + } + + Toggle(isOn: allEnabled) { + VStack(alignment: .leading) { + Text("All").foregroundColor(Color(.bravePrimary)) + } + } + #endif + ForEach($filterListStorage.filterLists) { $filterList in if !filterList.isHidden { Toggle(isOn: $filterList.isEnabled) { @@ -86,13 +99,6 @@ struct FilterListsView: View { .foregroundColor(Color(.secondaryBraveLabel)) } } - .onChange(of: filterList.isEnabled) { isEnabled in - if isEnabled { - expectedEnabledSources.insert(filterList.engineSource) - } else { - expectedEnabledSources.remove(filterList.engineSource) - } - } } } } @@ -129,12 +135,6 @@ struct FilterListsView: View { } } .onChange(of: filterListURL.setting.isEnabled) { isEnabled in - if isEnabled { - expectedEnabledSources.insert(filterListURL.setting.engineSource) - } else { - expectedEnabledSources.remove(filterListURL.setting.engineSource) - } - Task { CustomFilterListSetting.save(inMemory: !customFilterListStorage.persistChanges) } diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift index 4081beca165e..0b4d8a5e1b43 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/ContentBlockerScriptHandler.swift @@ -22,6 +22,11 @@ extension ContentBlockerHelper: TabContentScript { let data: [ContentblockerDTOData] } + enum BlockedType: Hashable { + case image + case ad + } + static let scriptName = "TrackingProtectionStats" static let scriptId = UUID().uuidString static let messageHandlerName = "\(scriptName)_\(messageUUID)" @@ -94,12 +99,12 @@ extension ContentBlockerHelper: TabContentScript { guard let domainURLString = domain.url else { return } let genericTypes = ContentBlockerManager.shared.validGenericTypes(for: domain) - let blockedType = await TPStatsBlocklistChecker.shared.blockedTypes( + let blockedType = await blockedTypes( requestURL: requestURL, sourceURL: sourceURL, enabledRuleTypes: genericTypes, resourceType: dto.resourceType, - isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + domain: domain ) guard let blockedType = blockedType else { return } @@ -148,4 +153,34 @@ extension ContentBlockerHelper: TabContentScript { Logger.module.error("\(error.localizedDescription)") } } + + @MainActor func blockedTypes( + requestURL: URL, + sourceURL: URL, + enabledRuleTypes: Set, + resourceType: AdblockEngine.ResourceType, + domain: Domain + ) async -> BlockedType? { + guard let host = requestURL.host, !host.isEmpty else { + // TP Stats init isn't complete yet + return nil + } + + if resourceType == .image && Preferences.Shields.blockImages.value { + return .image + } + + if enabledRuleTypes.contains(.blockAds) || enabledRuleTypes.contains(.blockTrackers) { + if await AdBlockGroupsManager.shared.shouldBlock( + requestURL: requestURL, + sourceURL: sourceURL, + resourceType: resourceType, + domain: domain + ) { + return .ad + } + } + + return nil + } } diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift index c14b994e2b84..c00ede362823 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/CosmeticFiltersScriptHandler.swift @@ -61,9 +61,9 @@ class CosmeticFiltersScriptHandler: TabContentScript { forUrl: frameURL, persistent: self.tab?.isPrivate == true ? false : true ) - let cachedEngines = await AdBlockStats.shared.cachedEngines(for: domain) + let cachedEngines = AdBlockGroupsManager.shared.cachedEngines(for: domain) - let selectorArrays = await cachedEngines.asyncConcurrentCompactMap { + let selectorArrays = await cachedEngines.asyncCompactMap { cachedEngine -> (selectors: Set, isAlwaysAggressive: Bool)? in do { guard @@ -76,7 +76,7 @@ class CosmeticFiltersScriptHandler: TabContentScript { return nil } - return (selectors, cachedEngine.isAlwaysAggressive) + return (selectors, cachedEngine.type.isAlwaysAggressive) } catch { Logger.module.error("\(error.localizedDescription)") return nil diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift index 9726d8744cc1..1194b5a72e84 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Paged/RequestBlockingContentScriptHandler.swift @@ -79,11 +79,11 @@ class RequestBlockingContentScriptHandler: TabContentScript { Task { @MainActor in let domain = Domain.getOrCreate(forUrl: currentTabURL, persistent: !isPrivateBrowsing) guard let domainURLString = domain.url else { return } - let shouldBlock = await AdBlockStats.shared.shouldBlock( + let shouldBlock = await AdBlockGroupsManager.shared.shouldBlock( requestURL: requestURL, sourceURL: sourceURL, resourceType: dto.data.resourceType, - isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + domain: domain ) // Ensure we check that the stats we're tracking is still for the same page diff --git a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift index f951989b87b7..55b67b36ea94 100644 --- a/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift +++ b/ios/brave-ios/Sources/Brave/Frontend/UserContent/UserScripts/Scripts_Dynamic/ScriptHandlers/Sandboxed/SiteStateListenerScriptHandler.swift @@ -77,7 +77,7 @@ class SiteStateListenerScriptHandler: TabContentScript { return } - let models = await AdBlockStats.shared.cosmeticFilterModels( + let models = await AdBlockGroupsManager.shared.cosmeticFilterModels( forFrameURL: frameURL, domain: domain ) @@ -102,7 +102,7 @@ class SiteStateListenerScriptHandler: TabContentScript { } @MainActor private func makeSetup( - from modelTuples: [AdBlockStats.CosmeticFilterModelTuple], + from modelTuples: [AdBlockGroupsManager.CosmeticFilterModelTuple], isAggressive: Bool ) throws -> UserScriptType.SelectorsPollerSetup { var standardSelectors: Set = [] diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift new file mode 100644 index 000000000000..7f345e33ff28 --- /dev/null +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockEngineManager.swift @@ -0,0 +1,421 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Data +import Foundation +import Preferences +import os + +/// A class for managing a single grouped engine +@MainActor class AdBlockEngineManager { + /// The directory to which we should store the unified list into + private static var cacheFolderDirectory: FileManager.SearchPathDirectory { + return FileManager.SearchPathDirectory.cachesDirectory + } + + public struct FileInfo: Hashable, Equatable { + let filterListInfo: GroupedAdBlockEngine.FilterListInfo + let localFileURL: URL + } + + /// The top level folder to create for caching engine data + private static let parentCacheFolderName = "engines" + /// All the info that is currently available + private var availableFiles: [FileInfo] + /// The subfolder that the cache data is stored. + /// Since we can have multiple engines this folder will be unique per engine + private let cacheFolderName: String + /// The type of engine this represents what kind of blocking we are doing. (aggressive or standard) + /// The difference is in how we treat first party blocking. Aggressive also blocks first party content. + let engineType: GroupedAdBlockEngine.EngineType + /// The engine that is currently available for this manager + private(set) var engine: GroupedAdBlockEngine? + /// This is a task that we use to delay a requested compile. + /// This allows us to wait for more sources and limit the number of times we compile + private var delayTask: Task? + /// This is the current pending group we are compiling. + /// This allows us to ensure we are not already compiling a newer version of the rules before setting the engine + private var pendingGroup: GroupedAdBlockEngine.FilterListGroup? + + /// This structure represents encodable info on what cached engine data contains + private struct CachedEngineInfo: Codable { + let infos: [GroupedAdBlockEngine.FilterListInfo] + let fileType: GroupedAdBlockEngine.FileType + } + + /// Get the already created cache folder + /// + /// - Note: Returns nil if the cache folder does not exist + private var createdCacheFolderURL: URL? { + guard let folderURL = Self.cacheFolderDirectory.url else { return nil } + let cacheFolderURL = folderURL.appendingPathComponent( + Self.parentCacheFolderName, + conformingTo: .folder + ) + .appendingPathComponent(cacheFolderName, conformingTo: .folder) + + if FileManager.default.fileExists(atPath: cacheFolderURL.path) { + return cacheFolderURL + } else { + return nil + } + } + + init(engineType: GroupedAdBlockEngine.EngineType, cacheFolderName: String) { + self.availableFiles = [] + self.engine = nil + self.engineType = engineType + self.cacheFolderName = cacheFolderName + } + + /// All the infos that are compilable based on the enabled sources and available infos + func compilableFiles( + for enabledSources: [GroupedAdBlockEngine.Source] + ) -> [FileInfo] { + return enabledSources.compactMap { source in + return availableFiles.first(where: { + FileManager.default.fileExists(atPath: $0.localFileURL.path) + && $0.filterListInfo.source == source + }) + } + } + + /// Add the info to the available list + func add(fileInfo: FileInfo) { + availableFiles.removeAll { existingFileInfo in + return existingFileInfo.filterListInfo == fileInfo.filterListInfo + } + + availableFiles.append(fileInfo) + } + + /// Remove any info from the available list given by the source + /// Mostly used for custom filter lists + func removeInfo(for source: GroupedAdBlockEngine.Source) { + availableFiles.removeAll { fileInfo in + return fileInfo.filterListInfo.source == source + } + } + + /// Checks to see if we need to compile or recompile the engine based on the available info + func checkNeedsCompile(for fileInfos: [AdBlockEngineManager.FileInfo]) -> Bool { + if let pendingGroup = pendingGroup { + return pendingGroup.infos != fileInfos.map({ $0.filterListInfo }) + } + + let compilableInfos = fileInfos.map({ $0.filterListInfo }) + guard !compilableInfos.isEmpty else { return engine?.group.infos.isEmpty == false } + guard let engine = engine else { return true } + return compilableInfos != engine.group.infos + } + + /// Load the engine from cache so it can be ready during launch + func loadFromCache(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo?) async -> Bool { + do { + guard let cachedGroupInfo = loadCachedInfo() else { return false } + + let engineType = self.engineType + let groupedEngine = try await Task.detached(priority: .high) { + let engine = try GroupedAdBlockEngine.compile( + group: cachedGroupInfo, + type: engineType + ) + + if let resourcesInfo = resourcesInfo { + try await engine.useResources(from: resourcesInfo) + } + + return engine + }.value + + self.set(engine: groupedEngine) + return true + } catch { + ContentBlockerManager.log.error( + "Failed to load engine from cache for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + + return false + } + } + + // Delete the cache. Mostly used for testing + func deleteCachedEngine() throws { + guard let cacheFolderURL = createdCacheFolderURL else { return } + try FileManager.default.removeItem(at: cacheFolderURL) + } + + /// This will compile available data, but will wait a little bit in case something new gets downloaded. + /// Especially needed during launch when we have a bunch of downloads coming at the same time. + func compileDelayedIfNeeded( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + ) { + // Cancel the previous task + delayTask?.cancel() + + // Restart the task + delayTask = Task { + let hasAllInfo = checkHasAllInfo(for: enabledSources) + try await Task.sleep(seconds: hasAllInfo ? 5 : 60) + + await compileAvailableIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo + ) + } + } + + /// This will compile available data right away if it is needed and cancel any delayedTasks + func compileImmediatelyIfNeeded( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + ) async { + delayTask?.cancel() + + await self.compileAvailableIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo + ) + } + + /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. + func update(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) async { + do { + try await engine?.useResources(from: resourcesInfo) + } catch { + ContentBlockerManager.log.error( + "Failed to update engine resources for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + } + } + + func needsCompile(for filterListInfo: GroupedAdBlockEngine.FilterListInfo) -> Bool { + var infos = availableFiles.map(\.filterListInfo) + if let engineInfos = engine?.group.infos { + // This is an optimization because during launch we don't have file infos. + // But we will have an engine loaded from cache. + // So we also check the engine infos as well + infos.append(contentsOf: engineInfos) + } + + guard let info = infos.first(where: { $0.source == filterListInfo.source }) else { + return true + } + + return filterListInfo.version < info.version + } + + func checkHasAllInfo(for sources: [GroupedAdBlockEngine.Source]) -> Bool { + let availableSources = compilableFiles(for: sources).map({ $0.filterListInfo.source }) + return sources.allSatisfy({ availableSources.contains($0) }) + } + + /// This will compile available data right away if it is needed + private func compileAvailableIfNeeded( + for enabledSources: [GroupedAdBlockEngine.Source], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + ) async { + do { + let compilableFiles = compilableFiles(for: enabledSources) + guard self.checkNeedsCompile(for: compilableFiles) else { return } + try await compileAvailable( + for: compilableFiles, + resourcesInfo: resourcesInfo + ) + } catch { + ContentBlockerManager.log.error( + "Failed to compile engine for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + } + } + + /// Compile an engine from all available data + private func compileAvailable( + for files: [AdBlockEngineManager.FileInfo], + resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + ) async throws { + let engineType = self.engineType + let group = try combineRules(for: files) + self.pendingGroup = group + + ContentBlockerManager.log.debug( + """ + Compiling `\(self.cacheFolderName)` engine from \(group.infos.count) sources: + \(group.debugDescription)" + """ + ) + + // 2. Compile the engine + let groupedEngine = try await Task.detached(priority: .high) { + let engine = try GroupedAdBlockEngine.compile(group: group, type: engineType) + + if let resourcesInfo { + try await engine.useResources(from: resourcesInfo) + } + + return engine + }.value + + // 3. Ensure our file is still up to date before setting it + // (avoid race conditiions) + guard pendingGroup == group else { return } + self.set(engine: groupedEngine) + await cache(engine: groupedEngine) + self.pendingGroup = nil + } + + private func set(engine: GroupedAdBlockEngine) { + let group = engine.group + ContentBlockerManager.log.debug( + """ + Set `\(self.cacheFolderName)` (\(group.fileType.debugDescription)) engine from \(group.infos.count) sources: + \(group.debugDescription)" + """ + ) + self.engine = engine + } + + /// Take all the filter lists and combine them into one then save them into a cache folder. + private func combineRules( + for compilableFiles: [AdBlockEngineManager.FileInfo] + ) throws -> GroupedAdBlockEngine.FilterListGroup { + // 1. Create a file url + let cachedFolder = try getOrCreateCacheFolder() + let fileURL = cachedFolder.appendingPathComponent("list.txt", conformingTo: .text) + var compiledInfos: [GroupedAdBlockEngine.FilterListInfo] = [] + var unifiedRules = "" + // 2. Join all the rules together + compilableFiles.forEach { fileInfo in + do { + let fileContents = try String(contentsOf: fileInfo.localFileURL) + compiledInfos.append(fileInfo.filterListInfo) + unifiedRules = [unifiedRules, fileContents].joined(separator: "\n") + } catch { + ContentBlockerManager.log.error( + "Could not load rules for \(fileInfo.filterListInfo.debugDescription): \(error)" + ) + } + } + + // 3. Save the files into storage + if FileManager.default.fileExists(atPath: fileURL.path) { + try FileManager.default.removeItem(at: fileURL) + } + try unifiedRules.write(to: fileURL, atomically: true, encoding: .utf8) + + // 4. Return a group containing info on this new file + return GroupedAdBlockEngine.FilterListGroup( + infos: compiledInfos, + localFileURL: fileURL, + fileType: .text + ) + } + + /// Get or create a cache folder for the given `Resource` + /// + /// - Note: This technically can't really return nil as the location and folder are hard coded + private func getOrCreateCacheFolder() throws -> URL { + guard + let folderURL = FileManager.default.getOrCreateFolder( + name: [Self.parentCacheFolderName, cacheFolderName].joined(separator: "/"), + location: Self.cacheFolderDirectory + ) + else { + throw ResourceFileError.failedToCreateCacheFolder + } + + return folderURL + } + + private func cache(engine: GroupedAdBlockEngine) async { + let encoder = JSONEncoder() + + do { + let folderURL = try getOrCreateCacheFolder() + + // Write the serialized engine + let serializedEngine = try await engine.serialize() + let serializedEngineURL = folderURL.appendingPathComponent("list.dat", conformingTo: .data) + + if FileManager.default.fileExists(atPath: serializedEngineURL.path) { + try FileManager.default.removeItem(at: serializedEngineURL) + } + + try serializedEngine.write(to: serializedEngineURL) + + // Write the info about the engine + let info = CachedEngineInfo(infos: engine.group.infos, fileType: .data) + let data = try encoder.encode(info) + let infoFileURL = folderURL.appendingPathComponent("engine_info.json", conformingTo: .json) + + if FileManager.default.fileExists(atPath: infoFileURL.path) { + try FileManager.default.removeItem(at: infoFileURL) + } + + try data.write(to: infoFileURL) + } catch { + ContentBlockerManager.log.error( + "Failed to save cache info for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + } + } + + private func loadCachedInfo() -> GroupedAdBlockEngine.FilterListGroup? { + guard let cacheFolderURL = createdCacheFolderURL else { return nil } + let cachedEngineURL = cacheFolderURL.appendingPathComponent("list.dat", conformingTo: .data) + guard FileManager.default.fileExists(atPath: cachedEngineURL.path) else { return nil } + let cachedInfoURL = cacheFolderURL.appendingPathComponent("engine_info", conformingTo: .json) + guard FileManager.default.fileExists(atPath: cachedInfoURL.path) else { return nil } + let decoder = JSONDecoder() + + do { + let data = try Data(contentsOf: cachedInfoURL) + let cachedInfo = try decoder.decode(CachedEngineInfo.self, from: data) + + return GroupedAdBlockEngine.FilterListGroup( + infos: cachedInfo.infos, + localFileURL: cachedEngineURL, + fileType: cachedInfo.fileType + ) + } catch { + ContentBlockerManager.log.error( + "Failed to load cache info for `\(self.cacheFolderName)`: \(String(describing: error))" + ) + return nil + } + } +} + +extension GroupedAdBlockEngine.Source { + func blocklistType(isAlwaysAggressive: Bool) -> ContentBlockerManager.BlocklistType? { + switch self { + case .filterList(let componentId, let uuid): + guard uuid != AdblockFilterListCatalogEntry.defaultFilterListComponentUUID else { + // For now we don't compile this into content blockers because we use the one coming from slim list + // We might change this in the future as it ends up with 95k items whereas the limit is 150k. + // So there is really no reason to use slim list except perhaps for performance which we need to test out. + return nil + } + + return .filterList(componentId: componentId, isAlwaysAggressive: isAlwaysAggressive) + case .filterListURL(let uuid): + return .customFilterList(uuid: uuid) + } + } +} + +extension AdblockFilterListCatalogEntry { + var engineSource: GroupedAdBlockEngine.Source { + return .filterList(componentId: componentId, uuid: uuid) + } +} + +extension CustomFilterListSetting { + @MainActor var engineSource: GroupedAdBlockEngine.Source { + return .filterListURL(uuid: uuid) + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift new file mode 100644 index 000000000000..f61be437d544 --- /dev/null +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/AdBlockGroupsManager.swift @@ -0,0 +1,427 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Data +import Foundation +import Preferences +import os + +/// A class that helps manage file sources and enabled filter lists so that 2 engines (standard and aggressive) are continually updated. +@MainActor public class AdBlockGroupsManager { + @MainActor protocol SourceProvider { + /// All the enabled sources. + /// They will be compiled in the order given so ensure the order here corresponds with how you want your engines to look + var enabledSources: [GroupedAdBlockEngine.Source] { get } + + /// If we didn't load the main engine from cache we need to load using the old cache mechanism + /// This is only temporary so we're not left with no ad-block during the upgrade. + /// We can drop all of this in future upgrades as by then we will have files cached in the new format + func legacyCacheFiles( + for engineType: GroupedAdBlockEngine.EngineType + ) -> [AdBlockEngineManager.FileInfo] + } + + typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) + public static let shared = AdBlockGroupsManager( + standardManager: GroupedAdBlockEngine.EngineType.standard.makeDefaultManager(), + aggressiveManager: GroupedAdBlockEngine.EngineType.aggressive.makeDefaultManager(), + contentBlockerManager: ContentBlockerManager.shared, + sourceProvider: DefaultSourceProvider() + ) + + private let standardManager: AdBlockEngineManager + private let aggressiveManager: AdBlockEngineManager + private let contentBlockerManager: ContentBlockerManager + private let sourceProvider: SourceProvider + + /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. + public var resourcesInfo: GroupedAdBlockEngine.ResourcesInfo? + + init( + standardManager: AdBlockEngineManager, + aggressiveManager: AdBlockEngineManager, + contentBlockerManager: ContentBlockerManager, + sourceProvider: SourceProvider + ) { + self.standardManager = standardManager + self.aggressiveManager = aggressiveManager + self.contentBlockerManager = contentBlockerManager + self.sourceProvider = sourceProvider + self.resourcesInfo = nil + } + + /// Handle memory warnings by freeing up some memory + func didReceiveMemoryWarning() async { + await standardManager.engine?.clearCaches() + await aggressiveManager.engine?.clearCaches() + } + + /// Load any cache data so its ready right during launch + func loadResourcesFromCache() async { + if let resourcesFolderURL = FilterListSetting.makeFolderURL( + forComponentFolderPath: Preferences.AppState.lastAdBlockResourcesFolderPath.value + ), FileManager.default.fileExists(atPath: resourcesFolderURL.path) { + // We need this for all filter lists so we can't compile anything until we download it + let resourcesInfo = getResourcesInfo(fromFolderURL: resourcesFolderURL) + self.resourcesInfo = resourcesInfo + + if #available(iOS 16.0, *) { + ContentBlockerManager.log.debug( + "Loaded resources component from cache: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" + ) + } else { + ContentBlockerManager.log.debug( + "Loaded resources component from cache: `\(resourcesInfo.localFileURL.path)`" + ) + } + } + } + + func loadEnginesFromCache() async { + await GroupedAdBlockEngine.EngineType.allCases.asyncConcurrentForEach { engineType in + await self.loadEngineFromCache(for: engineType) + } + } + + private func loadEngineFromCache(for engineType: GroupedAdBlockEngine.EngineType) async { + let manager = getManager(for: engineType) + + if await !manager.loadFromCache(resourcesInfo: self.resourcesInfo) { + // This migration will add ~24s on an iPhone 8 (~8s on an iPhone 14) + // Even though its a one time thing, let's skip it. + // We never waited for the aggressive engines to be ready before anyways + guard engineType == .standard else { return } + for fileInfo in sourceProvider.legacyCacheFiles(for: engineType) { + manager.add(fileInfo: fileInfo) + } + + await manager.compileImmediatelyIfNeeded( + for: sourceProvider.enabledSources, + resourcesInfo: self.resourcesInfo + ) + } + } + + /// Inform this manager of updates to the resources so our engines can be updated + func didUpdateResourcesComponent(folderURL: URL) async { + await Task { @MainActor in + let folderSubPath = FilterListSetting.extractFolderPath(fromComponentFolderURL: folderURL) + Preferences.AppState.lastAdBlockResourcesFolderPath.value = folderSubPath + }.value + + let version = folderURL.lastPathComponent + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: folderURL.appendingPathComponent("resources.json", conformingTo: .json), + version: version + ) + + updateIfNeeded(resourcesInfo: resourcesInfo) + } + + func update( + fileInfos: [AdBlockEngineManager.FileInfo], + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + let enabledSources = sourceProvider.enabledSources + + // Compile content blockers if this filter list is enabled + for fileInfo in fileInfos { + if enabledSources.contains(fileInfo.filterListInfo.source) { + Task { + await ensureContentBlockers(for: fileInfo, engineType: engineType) + } + } + + manager.add(fileInfo: fileInfo) + } + + manager.compileDelayedIfNeeded( + for: enabledSources, + resourcesInfo: resourcesInfo + ) + } + + /// Handle updated filter list info + func update( + fileInfo: AdBlockEngineManager.FileInfo, + engineType: GroupedAdBlockEngine.EngineType + ) { + update(fileInfos: [fileInfo], engineType: engineType) + } + + func removeFileInfos( + for sources: [GroupedAdBlockEngine.Source], + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + for source in sources { + manager.removeInfo(for: source) + } + + manager.compileDelayedIfNeeded( + for: sourceProvider.enabledSources, + resourcesInfo: resourcesInfo + ) + } + + func removeFileInfo( + for source: GroupedAdBlockEngine.Source, + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + manager.removeInfo(for: source) + + manager.compileDelayedIfNeeded( + for: sourceProvider.enabledSources, + resourcesInfo: resourcesInfo + ) + } + + /// Ensure all engines and content blockers are compiled + func compileEnginesIfNeeded() async { + let enabledSources = sourceProvider.enabledSources + await GroupedAdBlockEngine.EngineType.allCases.asyncConcurrentForEach { engineType in + let manager = self.getManager(for: engineType) + await manager.compileImmediatelyIfNeeded( + for: enabledSources, + resourcesInfo: self.resourcesInfo + ) + + self.ensureContentBlockers(for: enabledSources, engineType: engineType) + } + } + + /// Ensure all the content blockers are compiled for any file info found in the list of enabled sources + private func ensureContentBlockers( + for enabledSources: [GroupedAdBlockEngine.Source], + engineType: GroupedAdBlockEngine.EngineType + ) { + let manager = getManager(for: engineType) + // Compile all content blockers for the given manager + manager.compilableFiles(for: enabledSources).forEach { fileInfo in + Task { + await ensureContentBlockers(for: fileInfo, engineType: engineType) + } + } + } + + /// Ensure the content blocker is compiled for the given source + private func ensureContentBlockers( + for fileInfo: AdBlockEngineManager.FileInfo, + engineType: GroupedAdBlockEngine.EngineType + ) async { + let manager = getManager(for: engineType) + + guard + let blocklistType = fileInfo.filterListInfo.source.blocklistType( + isAlwaysAggressive: engineType.isAlwaysAggressive + ) + else { + return + } + + var modes = await contentBlockerManager.missingModes(for: blocklistType) + + if manager.needsCompile(for: fileInfo.filterListInfo) { + modes = blocklistType.allowedModes + } + + await contentBlockerManager.compileRuleList( + at: fileInfo.localFileURL, + for: blocklistType, + modes: modes + ) + } + + private func getResourcesInfo(fromFolderURL folderURL: URL) -> GroupedAdBlockEngine.ResourcesInfo + { + let version = folderURL.lastPathComponent + return GroupedAdBlockEngine.ResourcesInfo( + localFileURL: folderURL.appendingPathComponent("resources.json", conformingTo: .json), + version: version + ) + } + + /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. + func updateIfNeeded(resourcesInfo: GroupedAdBlockEngine.ResourcesInfo) { + guard self.resourcesInfo == nil || resourcesInfo.version > self.resourcesInfo!.version else { + return + } + self.resourcesInfo = resourcesInfo + + GroupedAdBlockEngine.EngineType.allCases.forEach { engineType in + let manager = self.getManager(for: engineType) + + Task { + await manager.update(resourcesInfo: resourcesInfo) + } + } + + if #available(iOS 16.0, *) { + ContentBlockerManager.log.debug( + "Updated resources component: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" + ) + } else { + ContentBlockerManager.log.debug( + "Updated resources component: `\(resourcesInfo.localFileURL.path)`" + ) + } + } + + /// Checks the general and regional engines to see if the request should be blocked + func shouldBlock( + requestURL: URL, + sourceURL: URL, + resourceType: AdblockEngine.ResourceType, + domain: Domain + ) async -> Bool { + return await cachedEngines(for: domain).asyncConcurrentMap({ cachedEngine in + return await cachedEngine.shouldBlock( + requestURL: requestURL, + sourceURL: sourceURL, + resourceType: resourceType, + isAggressiveMode: domain.blockAdsAndTrackingLevel.isAggressive + ) + }).contains(where: { $0 }) + } + + /// This returns all the user script types for the given frame + func makeEngineScriptTypes( + frameURL: URL, + isMainFrame: Bool, + isDeAmpEnabled: Bool, + domain: Domain + ) async -> Set { + // Add any engine scripts for this frame + return await cachedEngines(for: domain).enumerated().asyncMap({ + index, + cachedEngine -> Set in + do { + return try await cachedEngine.makeEngineScriptTypes( + frameURL: frameURL, + isMainFrame: isMainFrame, + isDeAmpEnabled: isDeAmpEnabled, + index: index + ) + } catch { + assertionFailure() + return [] + } + }).reduce( + Set(), + { partialResult, scriptTypes in + return partialResult.union(scriptTypes) + } + ) + } + + /// Returns all appropriate engines for the given domain + func cachedEngines(for domain: Domain) -> [GroupedAdBlockEngine] { + guard domain.isShieldExpected(.adblockAndTp, considerAllShieldsOption: true) else { return [] } + return GroupedAdBlockEngine.EngineType.allCases.compactMap({ getManager(for: $0).engine }) + } + + /// Returns all the models for this frame URL + func cosmeticFilterModels( + forFrameURL frameURL: URL, + domain: Domain + ) async -> [CosmeticFilterModelTuple] { + return await cachedEngines(for: domain).asyncConcurrentCompactMap { + cachedEngine -> CosmeticFilterModelTuple? in + do { + guard let model = try await cachedEngine.cosmeticFilterModel(forFrameURL: frameURL) else { + return nil + } + return (cachedEngine.type.isAlwaysAggressive, model) + } catch { + assertionFailure() + return nil + } + } + } + + private func getManager(for engineType: GroupedAdBlockEngine.EngineType) -> AdBlockEngineManager { + switch engineType { + case .standard: return standardManager + case .aggressive: return aggressiveManager + } + } +} + +extension GroupedAdBlockEngine.EngineType { + fileprivate var defaultCachedFolderName: String { + switch self { + case .standard: return "standard" + case .aggressive: return "aggressive" + } + } + + @MainActor fileprivate func makeDefaultManager() -> AdBlockEngineManager { + return AdBlockEngineManager(engineType: self, cacheFolderName: defaultCachedFolderName) + } +} + +extension FilterListSetting { + @MainActor var engineSource: GroupedAdBlockEngine.Source? { + guard let componentId = componentId else { return nil } + return .filterList(componentId: componentId, uuid: uuid) + } +} + +extension FilterList { + @MainActor var engineSource: GroupedAdBlockEngine.Source { + return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) + } +} + +extension AdBlockEngineManager.FileInfo { + init?( + for source: GroupedAdBlockEngine.Source, + downloadedFolderURL: URL + ) { + let version = downloadedFolderURL.lastPathComponent + let localFileURL = downloadedFolderURL.appendingPathComponent("list.txt") + + guard FileManager.default.fileExists(atPath: localFileURL.relativePath) else { + // We are loading the old component from cache. We don't want this file to be loaded. + // When we download the new component shortly we will update our cache. + // This should only trigger after an app update and eventually this check can be removed. + return nil + } + + self.init( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: source, + version: version + ), + localFileURL: localFileURL + ) + } +} + +@MainActor private class DefaultSourceProvider: AdBlockGroupsManager.SourceProvider { + /// Return an array of all sources that are enabled according to user's settings + /// - Note: This does not take into account the domain or global adblock toggle + var enabledSources: [GroupedAdBlockEngine.Source] { + var enabledSources = FilterListStorage.shared.enabledSources + enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) + return enabledSources + } + + func legacyCacheFiles( + for engineType: GroupedAdBlockEngine.EngineType + ) -> [AdBlockEngineManager.FileInfo] { + return FilterListStorage.shared.allFilterListSettings + .filter({ $0.isAlwaysAggressive == engineType.isAlwaysAggressive }) + .sorted(by: { $0.order?.intValue ?? 0 <= $1.order?.intValue ?? 0 }) + .compactMap({ setting in + guard let folderURL = setting.folderURL else { return nil } + guard let source = setting.engineSource else { return nil } + return AdBlockEngineManager.FileInfo(for: source, downloadedFolderURL: folderURL) + }) + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift similarity index 69% rename from ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift rename to ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift index 1e0c87f0f9d5..fca4404dcb4c 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdBlock/GroupedAdBlockEngine.swift @@ -1,4 +1,4 @@ -// Copyright 2022 The Brave Authors. All rights reserved. +// Copyright 2024 The Brave Authors. All rights reserved. // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -7,24 +7,24 @@ import BraveCore import Data import Foundation import Preferences -import os +import os.log /// An object that wraps around an `AdblockEngine` and caches some results /// and ensures information is always returned on the correct thread on the engine. -public actor CachedAdBlockEngine { - public enum Source: Hashable, CustomDebugStringConvertible { +public actor GroupedAdBlockEngine { + public enum Source: Codable, Hashable, CustomDebugStringConvertible { case filterList(componentId: String, uuid: String) case filterListURL(uuid: String) public var debugDescription: String { switch self { - case .filterList(let componentId, _): return "filterList(\(componentId))" - case .filterListURL(let uuid): return "filterListURL(\(uuid))" + case .filterList(let componentId, _): return componentId + case .filterListURL(let uuid): return uuid } } } - public enum FileType: Hashable, CustomDebugStringConvertible { + public enum FileType: Codable, Hashable, CustomDebugStringConvertible { case text, data public var debugDescription: String { @@ -35,14 +35,43 @@ public actor CachedAdBlockEngine { } } - public struct FilterListInfo: Hashable, Equatable, CustomDebugStringConvertible { - let source: Source - let localFileURL: URL + public enum EngineType: Hashable, CaseIterable, CustomDebugStringConvertible { + case standard + case aggressive + + var isAlwaysAggressive: Bool { + switch self { + case .standard: return false + case .aggressive: return true + } + } + + public var debugDescription: String { + switch self { + case .aggressive: return "aggressive" + case .standard: return "standard" + } + } + } + + public struct FilterListInfo: Codable, Hashable, Equatable, CustomDebugStringConvertible { + let source: GroupedAdBlockEngine.Source let version: String - let fileType: FileType public var debugDescription: String { - return "`\(source.debugDescription)` v\(version) (\(fileType.debugDescription))" + return "`\(source.debugDescription)` v\(version)" + } + } + + public struct FilterListGroup: Hashable, Equatable, CustomDebugStringConvertible { + let infos: [FilterListInfo] + let localFileURL: URL + let fileType: GroupedAdBlockEngine.FileType + + public var debugDescription: String { + return infos.enumerated() + .map({ " #\($0) \($1.debugDescription)" }) + .joined(separator: "\n") } } @@ -61,20 +90,14 @@ public actor CachedAdBlockEngine { private let engine: AdblockEngine - let isAlwaysAggressive: Bool - let filterListInfo: FilterListInfo - let resourcesInfo: ResourcesInfo + let type: EngineType + let group: FilterListGroup + private(set) var resourcesInfo: ResourcesInfo? - init( - engine: AdblockEngine, - filterListInfo: FilterListInfo, - resourcesInfo: ResourcesInfo, - isAlwaysAggressive: Bool - ) { + init(engine: AdblockEngine, group: FilterListGroup, type: EngineType) { self.engine = engine - self.filterListInfo = filterListInfo - self.resourcesInfo = resourcesInfo - self.isAlwaysAggressive = isAlwaysAggressive + self.group = group + self.type = type } /// Return the selectors that need to be hidden given the frameURL, ids and classes @@ -129,7 +152,7 @@ public actor CachedAdBlockEngine { requestURL: requestURL, sourceURL: sourceURL, resourceType: resourceType, - isAggressive: isAggressiveMode || self.isAlwaysAggressive + isAggressive: isAggressiveMode || self.type.isAlwaysAggressive ) cachedShouldBlockResult.addElement(shouldBlock, forKey: key) @@ -140,7 +163,6 @@ public actor CachedAdBlockEngine { func makeEngineScriptTypes( frameURL: URL, isMainFrame: Bool, - domain: Domain, isDeAmpEnabled: Bool, index: Int ) throws -> Set { @@ -175,6 +197,11 @@ public actor CachedAdBlockEngine { cachedFrameScriptTypes = FifoDict() } + func useResources(from info: ResourcesInfo) throws { + try engine.useResources(fromFileURL: info.localFileURL) + resourcesInfo = info + } + /// Serialize the engine into data to be later loaded from cache public func serialize() throws -> Data { return try engine.serialize() @@ -182,40 +209,32 @@ public actor CachedAdBlockEngine { /// Create an engine from the given resources public static func compile( - filterListInfo: FilterListInfo, - resourcesInfo: ResourcesInfo, - isAlwaysAggressive: Bool - ) throws -> CachedAdBlockEngine { + group: FilterListGroup, + type: EngineType + ) throws -> GroupedAdBlockEngine { let signpostID = Self.signpost.makeSignpostID() let state = Self.signpost.beginInterval( "compileEngine", id: signpostID, - "\(filterListInfo.debugDescription)" + "\(type.debugDescription) (\(group.fileType.debugDescription)): \(group.debugDescription)" ) do { - let engine = try makeEngine(from: filterListInfo) - try engine.useResources(fromFileURL: resourcesInfo.localFileURL) + let engine = try makeEngine(from: group) Self.signpost.endInterval("compileEngine", state) - - return CachedAdBlockEngine( - engine: engine, - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: isAlwaysAggressive - ) + return GroupedAdBlockEngine(engine: engine, group: group, type: type) } catch { Self.signpost.endInterval("compileEngine", state, "\(error.localizedDescription)") throw error } } - private static func makeEngine(from filterListInfo: FilterListInfo) throws -> AdblockEngine { - switch filterListInfo.fileType { + private static func makeEngine(from group: FilterListGroup) throws -> AdblockEngine { + switch group.fileType { case .data: - return try AdblockEngine(serializedData: Data(contentsOf: filterListInfo.localFileURL)) + return try AdblockEngine(serializedData: Data(contentsOf: group.localFileURL)) case .text: - return try AdblockEngine(rules: String(contentsOf: filterListInfo.localFileURL)) + return try AdblockEngine(rules: String(contentsOf: group.localFileURL)) } } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift b/ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift new file mode 100644 index 000000000000..6614354fd66b --- /dev/null +++ b/ios/brave-ios/Sources/Brave/WebFilters/AdblockFilterListCatalogEntry+Extensions.swift @@ -0,0 +1,30 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Foundation + +extension AdblockFilterListCatalogEntry { + /// This is the uuid of the default filter list. This is a special filter list which we use slim list content blockers for + public static let defaultFilterListComponentUUID = "default" + /// The component ID of the "Fanboy's Mobile Notifications List" + /// This is a special filter list that is enabled by default + public static let mobileAnnoyancesComponentID = "bfpgedeaaibpoidldhjcknekahbikncb" + /// The component id of the cookie consent notices filter list. + /// This is a special filter list that has more accessible UI to control it + public static let cookieConsentNoticesComponentID = "cdbbhgbmjhfnhnmgeddbliobbofkgdhe" + + public static let disabledFilterListComponentIDs = [ + // The Anti-porn list has 500251 rules and is strictly all content blocking driven content + // The limit for the rule store is 150000 rules. We have no way to handle this at the current moment + "lbnibkdpkdjnookgfeogjdanfenekmpe" + ] + + /// Lets us know if this filter list is always aggressive. + /// This value comes from `list_catalog.json` in brave core + var engineType: GroupedAdBlockEngine.EngineType { + return firstPartyProtections ? .standard : .aggressive + } +} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift index dba980798480..04033a2a326d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/ContentBlockerManager.swift @@ -70,6 +70,20 @@ actor ContentBlockerManager { } } + enum CompileResult { + case empty + case success(WKContentRuleList) + case failure(Error) + + func get() throws -> WKContentRuleList? { + switch self { + case .empty: return nil + case .success(let ruleList): return ruleList + case .failure(let error): throw error + } + } + } + /// An object representing the type of block list public enum BlocklistType: Hashable, CustomDebugStringConvertible { fileprivate static let genericPrifix = "stored-type" @@ -139,7 +153,7 @@ actor ContentBlockerManager { /// The store in which these rule lists should be compiled let ruleStore: WKContentRuleListStore /// We cached the rule lists so that we can return them quicker if we need to - private var cachedRuleLists: [String: Result] + private var cachedRuleLists: [String: CompileResult] /// A list of etld+1s that are always aggressive /// TODO: @JS Replace this with the 1st party ad-block list let alwaysAggressiveETLDs: Set = ["youtube.com"] @@ -184,27 +198,35 @@ actor ContentBlockerManager { /// Compile the rule list found in the given local URL using the specified modes func compileRuleList( at localFileURL: URL, - for type: BlocklistType, + for blocklistType: BlocklistType, modes: [BlockingMode] - ) async throws { + ) async { + guard !modes.isEmpty else { return } + let result: ContentBlockingRulesResult let signpostID = Self.signpost.makeSignpostID() let state = Self.signpost.beginInterval( "convertRules", id: signpostID, - "\(type.debugDescription)" + "\(blocklistType.debugDescription)" ) do { - let filterSet = try String(contentsOf: localFileURL) - result = try AdblockEngine.contentBlockerRules(fromFilterSet: filterSet) - Self.signpost.endInterval("convertRules", state) + do { + let filterSet = try String(contentsOf: localFileURL) + result = try AdblockEngine.contentBlockerRules(fromFilterSet: filterSet) + Self.signpost.endInterval("convertRules", state) + } catch { + Self.signpost.endInterval("convertRules", state, "\(error.localizedDescription)") + throw error + } + + try await compile(encodedContentRuleList: result.rulesJSON, for: blocklistType, modes: modes) } catch { - Self.signpost.endInterval("convertRules", state, "\(error.localizedDescription)") - throw error + ContentBlockerManager.log.error( + "Failed to compile rule list for `\(blocklistType.debugDescription)`" + ) } - - try await compile(encodedContentRuleList: result.rulesJSON, for: type, modes: modes) } /// Compile the given resource and store it in cache for the given blocklist type and specified modes @@ -215,13 +237,25 @@ actor ContentBlockerManager { ) async throws { guard !modes.isEmpty else { return } var foundError: Error? + let ruleList = try decode(encodedContentRuleList: encodedContentRuleList) + + guard !ruleList.isEmpty else { + for mode in modes { + self.cachedRuleLists[type.makeIdentifier(for: mode)] = .empty + } + + ContentBlockerManager.log.debug( + "Empty filter set for `\(type.debugDescription)`" + ) + return + } for mode in modes { let identifier = type.makeIdentifier(for: mode) do { let moddedRuleList = try self.modify( - encodedContentRuleList: encodedContentRuleList, + ruleList: ruleList, for: mode ) let ruleList = try await compile( @@ -229,6 +263,7 @@ actor ContentBlockerManager { for: type, mode: mode ) + self.cachedRuleLists[identifier] = .success(ruleList) Self.log.debug("Compiled rule list for `\(identifier)`") } catch { @@ -245,16 +280,18 @@ actor ContentBlockerManager { } } - private func modify(encodedContentRuleList: String, for mode: BlockingMode) throws -> String? { + private func modify( + ruleList: [[String: Any?]], + for mode: BlockingMode + ) throws -> String? { switch mode { case .aggressive, .general: // Aggressive mode and general mode has no modification to the rules - return nil + let modifiedData = try JSONSerialization.data(withJSONObject: ruleList) + return String(bytes: modifiedData, encoding: .utf8) case .standard: - // Add the ignore first party rule to make it standard - var ruleList = try decode(encodedContentRuleList: encodedContentRuleList) - + var ruleList = ruleList // We need to make sure we are not going over the limit // So we make space for the added rule if ruleList.count >= (Self.maxContentBlockerSize) { @@ -435,7 +472,7 @@ actor ContentBlockerManager { guard filterList.isEnabled else { return nil } return .filterList( componentId: filterList.entry.componentId, - isAlwaysAggressive: filterList.isAlwaysAggressive + isAlwaysAggressive: filterList.engineType.isAlwaysAggressive ) } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TPPageStats.swift similarity index 51% rename from ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift rename to ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TPPageStats.swift index a104edf7fe87..ea0d2f15faef 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TrackingProtectionPageStats.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ContentBlocker/TPPageStats.swift @@ -1,15 +1,9 @@ +// Copyright 2024 The Brave Authors. All rights reserved. // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// This file is largely verbatim from Focus iOS (Blockzilla/Lib/TrackingProtection). -// The preload and postload js files are unmodified from Focus. - -import BraveCore -import Data import Foundation -import Preferences -import Shared struct TPPageStats { let adCount: Int @@ -50,42 +44,3 @@ struct TPPageStats { ) } } - -class TPStatsBlocklistChecker { - static let shared = TPStatsBlocklistChecker() - - enum BlockedType: Hashable { - case image - case ad - } - - @MainActor func blockedTypes( - requestURL: URL, - sourceURL: URL, - enabledRuleTypes: Set, - resourceType: AdblockEngine.ResourceType, - isAggressiveMode: Bool - ) async -> BlockedType? { - guard let host = requestURL.host, !host.isEmpty else { - // TP Stats init isn't complete yet - return nil - } - - if resourceType == .image && Preferences.Shields.blockImages.value { - return .image - } - - if enabledRuleTypes.contains(.blockAds) || enabledRuleTypes.contains(.blockTrackers) { - if await AdBlockStats.shared.shouldBlock( - requestURL: requestURL, - sourceURL: sourceURL, - resourceType: resourceType, - isAggressiveMode: isAggressiveMode - ) { - return .ad - } - } - - return nil - } -} diff --git a/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift b/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift index b8a097125629..70d42cf3a3d4 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/CustomFilterListStorage.swift @@ -67,7 +67,7 @@ import Foundation extension CustomFilterListStorage { /// Gives us source representations of all the enabled custom filter lists - @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + @MainActor var enabledSources: [GroupedAdBlockEngine.Source] { return filterListsURLs .filter(\.setting.isEnabled) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift index e06be2fab019..feb1110e3437 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterList.swift @@ -7,21 +7,6 @@ import BraveCore import Foundation struct FilterList: Identifiable { - /// This is the uuid of the default filter list. This is a special filter list which we use slim list content blockers for - public static let defaultComponentUUID = "default" - /// The component ID of the "Fanboy's Mobile Notifications List" - /// This is a special filter list that is enabled by default - public static let mobileAnnoyancesComponentID = "bfpgedeaaibpoidldhjcknekahbikncb" - /// The component id of the cookie consent notices filter list. - /// This is a special filter list that has more accessible UI to control it - public static let cookieConsentNoticesComponentID = "cdbbhgbmjhfnhnmgeddbliobbofkgdhe" - /// This is a list of disabled filter lists. These lists are disabled because they are incompatible with iOS (for the time being) - public static let disabledComponentIDs = [ - // The Anti-porn list has 500251 rules and is strictly all content blocking driven content - // The limit for the rule store is 150000 rules. We have no way to handle this at the current moment - "lbnibkdpkdjnookgfeogjdanfenekmpe" - ] - var id: String { return entry.uuid } let order: Int let entry: AdblockFilterListCatalogEntry @@ -33,8 +18,8 @@ struct FilterList: Identifiable { /// Lets us know if this filter list is always aggressive. /// This value comes from `list_catalog.json` in brave core - var isAlwaysAggressive: Bool { - return !entry.firstPartyProtections + var engineType: GroupedAdBlockEngine.EngineType { + return entry.firstPartyProtections ? .standard : .aggressive } init(from entry: AdblockFilterListCatalogEntry, order: Int, isEnabled: Bool?) { diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift index 812e23e7c44f..02ce3094e50d 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift @@ -63,55 +63,22 @@ actor FilterListCustomURLDownloader: ObservableObject { } } - /// Handle the download results of a custom filter list. This will process the download by compiling iOS rule lists and adding the rule list to the `AdblockEngineManager`. + /// Handle the download results of a custom filter list. This will process the download by compiling iOS rule lists and adding the rule list to the `AdBlockEngineManager`. private func handle( downloadResult: ResourceDownloader.DownloadResult, for filterListCustomURL: FilterListCustomURL ) async { - let uuid = await filterListCustomURL.setting.uuid - - // Add/remove the resource depending on if it is enabled/disabled - guard let resourcesInfo = await AdBlockStats.shared.resourcesInfo else { - assertionFailure("This should not have been called if the resources are not ready") - return - } - let source = await filterListCustomURL.setting.engineSource let version = fileVersionDateFormatter.string(from: downloadResult.date) - let filterListInfo = CachedAdBlockEngine.FilterListInfo( - source: .filterListURL(uuid: uuid), - localFileURL: downloadResult.fileURL, - version: version, - fileType: .text - ) - let lazyInfo = AdBlockStats.LazyFilterListInfo( - filterListInfo: filterListInfo, - isAlwaysAggressive: true - ) - guard await AdBlockStats.shared.isEnabled(source: source) else { - await AdBlockStats.shared.updateIfNeeded( - filterListInfo: filterListInfo, - isAlwaysAggressive: true - ) - - // To free some space, remove any rule lists that are not needed - if let blocklistType = lazyInfo.blocklistType { - do { - try await ContentBlockerManager.shared.removeRuleLists(for: blocklistType) - } catch { - ContentBlockerManager.log.error( - "Failed to remove rule lists for \(filterListInfo.debugDescription)" - ) - } - } - return - } + let fileInfo = AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo(source: source, version: version), + localFileURL: downloadResult.fileURL + ) - await AdBlockStats.shared.compile( - lazyInfo: lazyInfo, - resourcesInfo: resourcesInfo, - compileContentBlockers: true + await AdBlockGroupsManager.shared.update( + fileInfo: fileInfo, + engineType: .aggressive ) } @@ -175,6 +142,11 @@ actor FilterListCustomURLDownloader: ObservableObject { let resource = await filterListCustomURL.setting.resource fetchTasks[resource]?.cancel() fetchTasks.removeValue(forKey: resource) + + await AdBlockGroupsManager.shared.removeFileInfo( + for: filterListCustomURL.setting.engineSource, + engineType: .aggressive + ) } } diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index 30870a884e34..b6b3ca8fb81c 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -32,68 +32,6 @@ public actor FilterListResourceDownloader { self.adBlockService = nil } - /// This loads the filter list settings from core data. - /// It uses these settings and other stored properties to load the enabled general shields and filter lists. - /// - /// - Warning: This method loads filter list settings. - /// You need to wait for `DataController.shared.initializeOnce()` to be called first before invoking this method - public func loadFilterListSettingsAndCachedData() async { - guard - let resourcesFolderURL = await FilterListSetting.makeFolderURL( - forComponentFolderPath: Preferences.AppState.lastAdBlockResourcesFolderPath.value - ), FileManager.default.fileExists(atPath: resourcesFolderURL.path) - else { - // We need this for all filter lists so we can't compile anything until we download it - return - } - - let resourcesInfo = await didUpdateResourcesComponent(folderURL: resourcesFolderURL) - await compileCachedFilterLists(resourcesInfo: resourcesInfo) - } - - /// This function adds engine resources to `AdBlockManager` from cached data representing the enabled filter lists. - /// - /// The filter lists are additional blocking content that can be added to the`AdBlockEngine` and as iOS content blockers. - /// It represents the items found in the "Filter lists" section of the "Shields" menu. - /// - /// - Note: The content blockers for these filter lists are not loaded at this point. They are cached by iOS and there is no need to reload them. - /// - /// - Warning: This method loads filter list settings. - /// You need to wait for `DataController.shared.initializeOnce()` to be called first before invoking this method - private func compileCachedFilterLists(resourcesInfo: CachedAdBlockEngine.ResourcesInfo) async { - await FilterListStorage.shared.loadFilterListSettings() - let filterListSettings = await FilterListStorage.shared.allFilterListSettings - - do { - try await filterListSettings.asyncConcurrentForEach { setting in - guard await setting.isEagerlyLoaded == true else { return } - guard let componentId = await setting.componentId else { return } - guard !FilterList.disabledComponentIDs.contains(componentId) else { return } - guard let source = await setting.engineSource else { return } - - // Try to load the filter list folder. We always have to compile this at start - guard let folderURL = await setting.folderURL, - FileManager.default.fileExists(atPath: folderURL.path) - else { - return - } - - await self.compileFilterListEngineIfNeeded( - source: source, - folderURL: folderURL, - isAlwaysAggressive: setting.isAlwaysAggressive, - resourcesInfo: resourcesInfo, - compileContentBlockers: false - ) - - // Sleep for 1ms. This drastically reduces memory usage without much impact to usability - try await Task.sleep(nanoseconds: 1_000_000) - } - } catch { - // Ignore the cancellation. - } - } - /// Start the adblock service to get updates to the `shieldsInstallPath` public func start(with adBlockService: AdblockService) { self.adBlockService = adBlockService @@ -110,7 +48,7 @@ public actor FilterListResourceDownloader { return } - await didUpdateResourcesComponent(folderURL: folderURL) + await AdBlockGroupsManager.shared.didUpdateResourcesComponent(folderURL: folderURL) await FilterListCustomURLDownloader.shared.startIfNeeded() if !FilterListStorage.shared.filterLists.isEmpty { @@ -122,9 +60,9 @@ public actor FilterListResourceDownloader { Task { @MainActor in for await filterListEntries in adBlockService.filterListCatalogComponentStream() { FilterListStorage.shared.loadFilterLists(from: filterListEntries) - ContentBlockerManager.log.debug("Loaded filter list catalog") - if await AdBlockStats.shared.resourcesInfo != nil { + + if AdBlockGroupsManager.shared.resourcesInfo != nil { await registerAllFilterListsIfNeeded(with: adBlockService) } } @@ -141,80 +79,24 @@ public actor FilterListResourceDownloader { } } - @discardableResult - /// When the - private func didUpdateResourcesComponent( - folderURL: URL - ) async -> CachedAdBlockEngine.ResourcesInfo { - await Task { @MainActor in - let folderSubPath = FilterListSetting.extractFolderPath(fromComponentFolderURL: folderURL) - Preferences.AppState.lastAdBlockResourcesFolderPath.value = folderSubPath - }.value - - let version = folderURL.lastPathComponent - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: folderURL.appendingPathComponent("resources.json"), - version: version - ) - - await AdBlockStats.shared.updateIfNeeded(resourcesInfo: resourcesInfo) - return resourcesInfo - } - /// Load general filter lists (shields) from the given `AdblockService` `shieldsInstallPath` `URL`. private func compileFilterListEngineIfNeeded( - source: CachedAdBlockEngine.Source, + source: GroupedAdBlockEngine.Source, folderURL: URL, - isAlwaysAggressive: Bool, - resourcesInfo: CachedAdBlockEngine.ResourcesInfo, - compileContentBlockers: Bool + engineType: GroupedAdBlockEngine.EngineType ) async { - let version = folderURL.lastPathComponent - let filterListURL = folderURL.appendingPathComponent("list.txt") - - guard FileManager.default.fileExists(atPath: filterListURL.relativePath) else { - // We are loading the old component from cache. We don't want this file to be loaded. - // When we download the new component shortly we will update our cache. - // This should only trigger after an app update and eventually this check can be removed. - return - } - - let filterListInfo = CachedAdBlockEngine.FilterListInfo( - source: source, - localFileURL: filterListURL, - version: version, - fileType: .text - ) - let lazyInfo = AdBlockStats.LazyFilterListInfo( - filterListInfo: filterListInfo, - isAlwaysAggressive: isAlwaysAggressive - ) - - // Check if we should load these rules - guard await AdBlockStats.shared.isEnabled(source: source) else { - await AdBlockStats.shared.updateIfNeeded( - filterListInfo: filterListInfo, - isAlwaysAggressive: isAlwaysAggressive + guard + let fileInfo = AdBlockEngineManager.FileInfo( + for: source, + downloadedFolderURL: folderURL ) - - // To free some space, remove any rule lists that are not needed - if let blocklistType = lazyInfo.blocklistType { - do { - try await ContentBlockerManager.shared.removeRuleLists(for: blocklistType) - } catch { - ContentBlockerManager.log.error( - "Failed to remove rule lists for \(filterListInfo.debugDescription)" - ) - } - } - + else { return } - await AdBlockStats.shared.compile( - lazyInfo: lazyInfo, - resourcesInfo: resourcesInfo, - compileContentBlockers: compileContentBlockers + await AdBlockGroupsManager.shared.update( + fileInfo: fileInfo, + engineType: engineType ) } @@ -227,19 +109,13 @@ public actor FilterListResourceDownloader { adBlockServiceTasks[filterList.entry.componentId] = Task { @MainActor in for await folderURL in adBlockService.register(filterList: filterList) { guard let folderURL = folderURL else { continue } - guard let resourcesInfo = await AdBlockStats.shared.resourcesInfo else { - assertionFailure("We shouldn't have started downloads before getting this value") - return - } let source = filterList.engineSource // Add or remove the filter list from the engine depending if it's been enabled or not await self.compileFilterListEngineIfNeeded( source: source, folderURL: folderURL, - isAlwaysAggressive: filterList.isAlwaysAggressive, - resourcesInfo: resourcesInfo, - compileContentBlockers: true + engineType: filterList.engineType ) // Save the downloaded folder for later (caching) purposes diff --git a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift index dca5cf0d70d9..148d237aea0f 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/FilterListStorage.swift @@ -56,7 +56,11 @@ import Preferences index, adBlockFilterList -> FilterList? in // Certain filter lists are disabled if they are currently incompatible with iOS - guard !FilterList.disabledComponentIDs.contains(adBlockFilterList.componentId) else { + guard + !AdblockFilterListCatalogEntry.disabledFilterListComponentIDs.contains( + adBlockFilterList.componentId + ) + else { return nil } let setting = allFilterListSettings.first(where: { @@ -125,7 +129,9 @@ import Preferences /// - Warning: Do not call this before we load core data public func isEnabled(for componentId: String) -> Bool { - guard !FilterList.disabledComponentIDs.contains(componentId) else { return false } + guard !AdblockFilterListCatalogEntry.disabledFilterListComponentIDs.contains(componentId) else { + return false + } return filterLists.first(where: { $0.entry.componentId == componentId })?.isEnabled ?? allFilterListSettings.first(where: { $0.componentId == componentId })?.isEnabled @@ -154,7 +160,7 @@ import Preferences componentId: filterList.entry.componentId, allowCreation: true, order: filterList.order, - isAlwaysAggressive: filterList.isAlwaysAggressive, + isAlwaysAggressive: filterList.engineType.isAlwaysAggressive, isDefaultEnabled: filterList.entry.defaultEnabled ) } @@ -276,7 +282,7 @@ import Preferences Task { @MainActor in UmaHistogramBoolean( "Brave.Shields.CookieListEnabled", - isEnabled(for: FilterList.cookieConsentNoticesComponentID) + isEnabled(for: AdblockFilterListCatalogEntry.cookieConsentNoticesComponentID) ) } } @@ -295,10 +301,11 @@ extension AdblockFilterListCatalogEntry { extension FilterListStorage { /// Gives us source representations of all the enabled filter lists - @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + @MainActor var enabledSources: [GroupedAdBlockEngine.Source] { return filterLists.isEmpty ? allFilterListSettings .filter(\.isEnabled) + .sorted(by: { $0.order?.intValue ?? 0 <= $1.order?.intValue ?? 0 }) .compactMap(\.engineSource) : filterLists .filter(\.isEnabled) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift b/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift index 6c17f0080705..36159c14d91f 100644 --- a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift +++ b/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift @@ -13,7 +13,7 @@ extension AdblockEngine { case couldNotDeserializeDATFile } - public func useResources(fromFileURL fileURL: URL) throws { + func useResources(fromFileURL fileURL: URL) throws { // Add scriplets if available if let json = try Self.validateJSON(Data(contentsOf: fileURL)) { useResources(json) diff --git a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift b/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift deleted file mode 100644 index 8ef7dc579624..000000000000 --- a/ios/brave-ios/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift +++ /dev/null @@ -1,355 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import BraveCore -import BraveShared -import Combine -import Data -import Foundation -import Shared -import os.log - -/// This object holds on to our adblock engines and returns information needed for stats tracking as well as some conveniences -/// for injected scripts needed during web navigation and cosmetic filters models needed by the `SelectorsPollerScript.js` script. -public actor AdBlockStats { - typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) - public static let shared = AdBlockStats() - - /// An object containing the basic information to allow us to compile an engine - public struct LazyFilterListInfo { - let filterListInfo: CachedAdBlockEngine.FilterListInfo - let isAlwaysAggressive: Bool - } - - /// A list of filter list info that are available for compilation. This information is used for lazy loading. - private(set) var availableFilterLists: [CachedAdBlockEngine.Source: LazyFilterListInfo] - /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. - public private(set) var resourcesInfo: CachedAdBlockEngine.ResourcesInfo? - /// Adblock engine for general adblock lists. - private(set) var cachedEngines: [CachedAdBlockEngine.Source: CachedAdBlockEngine] - /// The current task that is compiling. - private var currentCompileTask: Task<(), Never>? - - /// Return an array of all sources that are enabled according to user's settings - /// - Note: This does not take into account the domain or global adblock toggle - @MainActor var enabledSources: [CachedAdBlockEngine.Source] { - var enabledSources = FilterListStorage.shared.enabledSources - enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) - return enabledSources - } - - init() { - cachedEngines = [:] - availableFilterLists = [:] - } - - /// Handle memory warnings by freeing up some memory - func didReceiveMemoryWarning() async { - await cachedEngines.values.asyncForEach({ await $0.clearCaches() }) - await removeDisabledEngines() - } - - /// Create and add an engine from the given resources. - /// If an engine already exists for the given source, it will be replaced. - public func compile( - lazyInfo: LazyFilterListInfo, - resourcesInfo: CachedAdBlockEngine.ResourcesInfo, - compileContentBlockers: Bool - ) async { - await currentCompileTask?.value - - currentCompileTask = Task.detached { - // Compile engine - if await self.needsCompilation(for: lazyInfo.filterListInfo, resourcesInfo: resourcesInfo) { - do { - let engine = try CachedAdBlockEngine.compile( - filterListInfo: lazyInfo.filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: lazyInfo.isAlwaysAggressive - ) - - await self.add(engine: engine) - } catch { - ContentBlockerManager.log.error( - "Failed to compile engine for \(lazyInfo.filterListInfo.source.debugDescription)" - ) - } - } - - // Compile content blockers - if compileContentBlockers, let blocklistType = lazyInfo.blocklistType { - let modes = await ContentBlockerManager.shared.missingModes(for: blocklistType) - guard !modes.isEmpty else { return } - - do { - try await ContentBlockerManager.shared.compileRuleList( - at: lazyInfo.filterListInfo.localFileURL, - for: blocklistType, - modes: modes - ) - } catch { - ContentBlockerManager.log.error( - "Failed to compile rule list for \(lazyInfo.filterListInfo.source.debugDescription)" - ) - } - } - } - - await currentCompileTask?.value - } - - /// Add a new engine to the list. - /// If an engine already exists for the same source, it will be replaced instead. - private func add(engine: CachedAdBlockEngine) { - cachedEngines[engine.filterListInfo.source] = engine - updateIfNeeded(resourcesInfo: engine.resourcesInfo) - updateIfNeeded( - filterListInfo: engine.filterListInfo, - isAlwaysAggressive: engine.isAlwaysAggressive - ) - let typeString = engine.isAlwaysAggressive ? "aggressive" : "standard" - ContentBlockerManager.log.debug( - "Added \(typeString) engine for \(engine.filterListInfo.debugDescription)" - ) - } - - /// Add or update `filterListInfo` if it is a newer version. This information is used for lazy loading. - func updateIfNeeded(filterListInfo: CachedAdBlockEngine.FilterListInfo, isAlwaysAggressive: Bool) - { - if let existingLazyInfo = availableFilterLists[filterListInfo.source] { - guard filterListInfo.version > existingLazyInfo.filterListInfo.version else { return } - } - - availableFilterLists[filterListInfo.source] = LazyFilterListInfo( - filterListInfo: filterListInfo, - isAlwaysAggressive: isAlwaysAggressive - ) - } - - /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. - func updateIfNeeded(resourcesInfo: CachedAdBlockEngine.ResourcesInfo) { - guard self.resourcesInfo == nil || resourcesInfo.version > self.resourcesInfo!.version else { - return - } - self.resourcesInfo = resourcesInfo - - if #available(iOS 16.0, *) { - ContentBlockerManager.log.debug( - "Updated resources component: `\(resourcesInfo.localFileURL.path(percentEncoded: false))`" - ) - } else { - ContentBlockerManager.log.debug( - "Updated resources component: `\(resourcesInfo.localFileURL.path)`" - ) - } - } - - /// Remove all the engines - func removeAllEngines() { - cachedEngines.removeAll() - } - - /// Remove all engines that have disabled sources - func removeDisabledEngines() async { - let sources = await Set(enabledSources) - - for source in cachedEngines.keys { - guard !sources.contains(source) else { continue } - // Remove the engine - if let filterListInfo = cachedEngines[source]?.filterListInfo { - cachedEngines.removeValue(forKey: source) - ContentBlockerManager.log.debug("Removed engine for \(filterListInfo.debugDescription)") - } - - // Delete the Content blockers - if let lazyInfo = availableFilterLists[source], let blocklistType = lazyInfo.blocklistType { - do { - try await ContentBlockerManager.shared.removeRuleLists(for: blocklistType) - } catch { - ContentBlockerManager.log.error( - "Failed to remove rule lists for \(lazyInfo.filterListInfo.debugDescription)" - ) - } - } - } - } - - /// Remove all engines that have disabled sources - func ensureEnabledEngines() async { - do { - for source in await enabledSources { - guard cachedEngines[source] == nil else { continue } - guard let availableFilterList = availableFilterLists[source] else { continue } - guard let resourcesInfo = self.resourcesInfo else { continue } - - await compile( - lazyInfo: availableFilterList, - resourcesInfo: resourcesInfo, - compileContentBlockers: true - ) - - // Sleep for 1ms. This drastically reduces memory usage without much impact to usability - try await Task.sleep(nanoseconds: 1_000_000) - } - } catch { - // Ignore cancellation errors - } - } - - /// Tells us if this source should be loaded. - @MainActor func isEnabled(source: CachedAdBlockEngine.Source) -> Bool { - return enabledSources.contains(source) - } - - /// Tells us if an engine needs compilation if it's missing or if its resources are outdated - func needsCompilation( - for filterListInfo: CachedAdBlockEngine.FilterListInfo, - resourcesInfo: CachedAdBlockEngine.ResourcesInfo - ) -> Bool { - if let cachedEngine = cachedEngines[filterListInfo.source] { - return cachedEngine.filterListInfo.version < filterListInfo.version - && cachedEngine.resourcesInfo.version < resourcesInfo.version - } else { - return true - } - } - - /// Checks the general and regional engines to see if the request should be blocked - func shouldBlock( - requestURL: URL, - sourceURL: URL, - resourceType: AdblockEngine.ResourceType, - isAggressiveMode: Bool - ) async -> Bool { - let sources = await self.enabledSources - return await cachedEngines(for: sources).asyncConcurrentMap({ cachedEngine in - return await cachedEngine.shouldBlock( - requestURL: requestURL, - sourceURL: sourceURL, - resourceType: resourceType, - isAggressiveMode: isAggressiveMode - ) - }).contains(where: { $0 }) - } - - /// This returns all the user script types for the given frame - func makeEngineScriptTypes( - frameURL: URL, - isMainFrame: Bool, - isDeAmpEnabled: Bool, - domain: Domain - ) async -> Set { - // Add any engine scripts for this frame - return await cachedEngines(for: domain).enumerated().asyncMap({ - index, - cachedEngine -> Set in - do { - return try await cachedEngine.makeEngineScriptTypes( - frameURL: frameURL, - isMainFrame: isMainFrame, - domain: domain, - isDeAmpEnabled: isDeAmpEnabled, - index: index - ) - } catch { - assertionFailure() - return [] - } - }).reduce( - Set(), - { partialResult, scriptTypes in - return partialResult.union(scriptTypes) - } - ) - } - - /// Returns all appropriate engines for the given domain - @MainActor func cachedEngines(for domain: Domain) async -> [CachedAdBlockEngine] { - let sources = enabledSources(for: domain) - return await cachedEngines(for: sources) - } - - /// Return all the cached engines for the given sources. If any filter list is not yet loaded, it will be lazily loaded - private func cachedEngines(for sources: [CachedAdBlockEngine.Source]) -> [CachedAdBlockEngine] { - return sources.compactMap { source -> CachedAdBlockEngine? in - return cachedEngines[source] - } - } - - /// Returns all the models for this frame URL - func cosmeticFilterModels( - forFrameURL frameURL: URL, - domain: Domain - ) async -> [CosmeticFilterModelTuple] { - return await cachedEngines(for: domain).asyncConcurrentCompactMap { - cachedEngine -> CosmeticFilterModelTuple? in - do { - guard let model = try await cachedEngine.cosmeticFilterModel(forFrameURL: frameURL) else { - return nil - } - return (cachedEngine.isAlwaysAggressive, model) - } catch { - assertionFailure() - return nil - } - } - } - - /// Give us all the enabled sources for the given domain - @MainActor private func enabledSources(for domain: Domain) -> [CachedAdBlockEngine.Source] { - let enabledSources = self.enabledSources - return enabledSources.filter({ $0.isEnabled(for: domain) }) - } -} - -extension FilterListSetting { - @MainActor var engineSource: CachedAdBlockEngine.Source? { - guard let componentId = componentId else { return nil } - return .filterList(componentId: componentId, uuid: uuid) - } -} - -extension FilterList { - @MainActor var engineSource: CachedAdBlockEngine.Source { - return .filterList(componentId: entry.componentId, uuid: self.entry.uuid) - } -} - -extension CustomFilterListSetting { - @MainActor var engineSource: CachedAdBlockEngine.Source { - return .filterListURL(uuid: uuid) - } -} - -extension CachedAdBlockEngine.Source { - /// Returns a boolean indicating if the engine is enabled for the given domain. - /// - /// This is determined by checking the source of the engine and checking the appropriate shields. - @MainActor fileprivate func isEnabled(for domain: Domain) -> Bool { - switch self { - case .filterList, .filterListURL: - // This engine source type is enabled only if shields are enabled - // for the given domain - return domain.isShieldExpected(.adblockAndTp, considerAllShieldsOption: true) - } - } -} - -extension AdBlockStats.LazyFilterListInfo { - var blocklistType: ContentBlockerManager.BlocklistType? { - switch filterListInfo.source { - case .filterList(let componentId, let uuid): - guard uuid != FilterList.defaultComponentUUID else { - // For now we don't compile this into content blockers because we use the one coming from slim list - // We might change this in the future as it ends up with 95k items whereas the limit is 150k. - // So there is really no reason to use slim list except perhaps for performance which we need to test out. - return nil - } - - return .filterList(componentId: componentId, isAlwaysAggressive: isAlwaysAggressive) - case .filterListURL(let uuid): - return .customFilterList(uuid: uuid) - } - } -} diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift new file mode 100644 index 000000000000..e4a2002a3fb9 --- /dev/null +++ b/ios/brave-ios/Tests/ClientTests/AdBlockEngineManagerTests.swift @@ -0,0 +1,120 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import XCTest + +@testable import Brave + +final class AdBlockEngineManagerTests: XCTestCase { + func testEngineManager() async throws { + // Given + // Engine manager and file info and resources info + let engineManager = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test-standard" + ) + let sampleFilterListURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", + withExtension: "txt" + )! + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + version: "0" + ) + let fileInfos = [ + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: sampleFilterListURL + ), + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: sampleFilterListURL + ), + ] + let sources = fileInfos.map({ $0.filterListInfo.source }) + + // When + // Adding file info + for fileInfo in fileInfos { + await engineManager.add(fileInfo: fileInfo) + } + + // Then + // Needs compile returns true and there is no engine + var needsCompile = await engineManager.checkNeedsCompile(for: fileInfos) + var engine = await engineManager.engine + XCTAssertTrue(needsCompile) + XCTAssertNil(engine) + + // When + // Loaded from cache + var loadedFromCache = await engineManager.loadFromCache(resourcesInfo: resourcesInfo) + + // Then + // We don't have any engine + engine = await engineManager.engine + XCTAssertNil(engine) + XCTAssertFalse(loadedFromCache) + + // When 2 + // We compile engine + await engineManager.compileImmediatelyIfNeeded( + for: sources, + resourcesInfo: resourcesInfo + ) + + // Then + // Needs compile returns false and engine is correctly created + needsCompile = await engineManager.checkNeedsCompile(for: fileInfos) + engine = await engineManager.engine + let compiledResources = await engine?.resourcesInfo + let group = await engine?.group + XCTAssertFalse(needsCompile) + XCTAssertNotNil(engine) + XCTAssertEqual(group?.infos, fileInfos.map({ $0.filterListInfo })) + XCTAssertEqual(group?.fileType, .text) + XCTAssertEqual(compiledResources, resourcesInfo) + + // When + // We load from cache using another manager with the same cache folder name + let engineManager2 = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test-standard" + ) + loadedFromCache = await engineManager2.loadFromCache(resourcesInfo: resourcesInfo) + + // Then + // We have a cached engine + engine = await engineManager2.engine + let cacheGroup = await engine?.group + let expectedURL = group?.localFileURL.deletingPathExtension().appendingPathExtension("dat") + XCTAssertTrue(loadedFromCache) + XCTAssertNotNil(engine) + XCTAssertEqual(cacheGroup?.localFileURL, expectedURL) + XCTAssertEqual(cacheGroup?.infos, fileInfos.map({ $0.filterListInfo })) + XCTAssertEqual(cacheGroup?.fileType, .data) + try await engineManager2.deleteCachedEngine() + + // When + // We load from cache using a third manager with the same cache folder name + let engineManager3 = await AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test-standard" + ) + loadedFromCache = await engineManager3.loadFromCache(resourcesInfo: resourcesInfo) + + // Then + // There is no cached engine because we deleted it previously + engine = await engineManager3.engine + XCTAssertFalse(loadedFromCache) + XCTAssertNil(engine) + } +} diff --git a/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift new file mode 100644 index 000000000000..aea730f0c445 --- /dev/null +++ b/ios/brave-ios/Tests/ClientTests/AdBlockGroupsManagerTests.swift @@ -0,0 +1,272 @@ +// Copyright 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import BraveCore +import Data +import WebKit +import XCTest + +@testable import Brave + +final class AdBlockGroupsManagerTests: XCTestCase { + private static var sampleFilterListURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", + withExtension: "txt" + )! + + @MainActor private lazy var ruleStore: WKContentRuleListStore = { + let testBundle = Bundle.module + let bundleURL = testBundle.bundleURL + return WKContentRuleListStore(url: bundleURL)! + }() + + /// Testing engine compilations and filter list managment + @MainActor func testCompilation() async throws { + AdblockEngine.setDomainResolver() + + // Given + // A source provider and groups manager + let fileInfos = [ + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: Self.sampleFilterListURL + ), + AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: Self.sampleFilterListURL + ), + ] + let sourceProvider = TestSourceProvider(fileInfos: fileInfos) + let standardManager = AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ) + let aggressiveManager = AdBlockEngineManager( + engineType: .aggressive, + cacheFolderName: "test_aggressive" + ) + let groupsManager = AdBlockGroupsManager( + standardManager: standardManager, + aggressiveManager: aggressiveManager, + contentBlockerManager: makeContentBlockingManager(), + sourceProvider: sourceProvider + ) + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + version: "0" + ) + + // When + // Adding file info and enabling sources for only one engine + sourceProvider.set( + source: fileInfos[0].filterListInfo.source, + enabled: true + ) + groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + groupsManager.update(fileInfo: fileInfos[1], engineType: .standard) + groupsManager.update(fileInfos: fileInfos, engineType: .aggressive) + await groupsManager.compileEnginesIfNeeded() + + // Then + // Only one engine is created + var standardEngine = standardManager.engine + var aggressiveEngine = aggressiveManager.engine + var standardGroup = await standardEngine?.group + var aggressiveGroup = await aggressiveEngine?.group + var standardResources = await standardEngine?.resourcesInfo + var aggressiveResources = await aggressiveEngine?.resourcesInfo + XCTAssertNil(standardEngine) + XCTAssertNotNil(aggressiveEngine) + XCTAssertEqual(aggressiveResources, resourcesInfo) + XCTAssertEqual(aggressiveGroup?.infos, [fileInfos[0].filterListInfo]) + + // When + // We enable sources and recompile the engine + sourceProvider.set( + source: fileInfos[1].filterListInfo.source, + enabled: true + ) + groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + await groupsManager.compileEnginesIfNeeded() + + // Then + // All engines are created + standardEngine = standardManager.engine + aggressiveEngine = aggressiveManager.engine + standardGroup = await standardEngine?.group + aggressiveGroup = await aggressiveEngine?.group + standardResources = await standardEngine?.resourcesInfo + aggressiveResources = await aggressiveEngine?.resourcesInfo + XCTAssertNotNil(standardEngine) + XCTAssertNotNil(aggressiveEngine) + XCTAssertEqual(standardResources, resourcesInfo) + XCTAssertEqual(aggressiveResources, resourcesInfo) + XCTAssertEqual(standardGroup?.infos, [fileInfos[1].filterListInfo]) + XCTAssertEqual(aggressiveGroup?.infos, fileInfos.map({ $0.filterListInfo })) + } + + /// Partially testing expectations found in https://dev-pages.bravesoftware.com/filtering/index.html + @MainActor func testBlocking() async throws { + AdblockEngine.setDomainResolver() + + // Given + // A source provider and groups manager + let fileInfo = AdBlockEngineManager.FileInfo( + filterListInfo: GroupedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: UUID().uuidString), + version: "0" + ), + localFileURL: Self.sampleFilterListURL + ) + let sourceProvider = TestSourceProvider(fileInfos: [fileInfo]) + let standardManager = AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ) + let aggressiveManager = AdBlockEngineManager( + engineType: .aggressive, + cacheFolderName: "test_aggressive" + ) + let groupsManager = AdBlockGroupsManager( + standardManager: standardManager, + aggressiveManager: aggressiveManager, + contentBlockerManager: makeContentBlockingManager(), + sourceProvider: sourceProvider + ) + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + version: "0" + ) + + // When + // Adding creating a standard engine + sourceProvider.set( + source: fileInfo.filterListInfo.source, + enabled: true + ) + groupsManager.updateIfNeeded(resourcesInfo: resourcesInfo) + groupsManager.update(fileInfo: fileInfo, engineType: .standard) + await groupsManager.compileEnginesIfNeeded() + + let mainFrameURL = URL(string: "https://dev-pages.bravesoftware.com")! + let domain = await MainActor.run { + return Domain.getOrCreate(forUrl: mainFrameURL, persistent: false) + } + + // Then + // Should have certian script types + let scriptTypes = await groupsManager.makeEngineScriptTypes( + frameURL: URL(string: "https://dev-pages.bravesoftware.com/filtering/scriptlets.html")!, + isMainFrame: true, + isDeAmpEnabled: false, + domain: domain + ) + XCTAssertTrue( + scriptTypes.contains(where: { scriptType in + switch scriptType { + case .engineScript(let configuration): + return configuration.source.contains("window.BRAVE_TEST_VALUE") + default: + return false + } + }) + ) + + // Then + // Should return some filter models + let cosmeticFilterModels = await groupsManager.cosmeticFilterModels( + forFrameURL: mainFrameURL, + domain: domain + ) + XCTAssertFalse(cosmeticFilterModels.isEmpty) + + // Should block certain 3rd party content + let sourceURL = URL( + string: "https://dev-pages.bravesoftware.com/filtering/network-requests.html" + )! + let requestURL = URL( + string: "https://dev-pages.brave.software/static/images/test.jpg?335962573013224749" + )! + var blockResult = await groupsManager.shouldBlock( + requestURL: requestURL, + sourceURL: sourceURL, + resourceType: .image, + domain: domain + ) + XCTAssertTrue(blockResult) + + // Should not block certain 1sd party content + let requestURL2 = URL( + string: "https://dev-pages.bravesoftware.com/static/images/test.jpg?335962573013224749" + )! + blockResult = await groupsManager.shouldBlock( + requestURL: requestURL2, + sourceURL: sourceURL, + resourceType: .image, + domain: domain + ) + XCTAssertFalse(blockResult) + + // When + // Adding an aggressive filter list + groupsManager.update(fileInfo: fileInfo, engineType: .aggressive) + await groupsManager.compileEnginesIfNeeded() + + // Then + // Should block certain 1sd party content + blockResult = await groupsManager.shouldBlock( + requestURL: requestURL2, + sourceURL: sourceURL, + resourceType: .image, + domain: domain + ) + XCTAssertTrue(blockResult) + } + + @MainActor private func makeContentBlockingManager() -> ContentBlockerManager { + return ContentBlockerManager(ruleStore: ruleStore) + } +} + +class TestSourceProvider: AdBlockGroupsManager.SourceProvider { + /// The sources that are enabled + private var _enabledSources: Set + private var fileInfos: [AdBlockEngineManager.FileInfo] = [] + + init( + _enabledSources: Set = [], + fileInfos: [AdBlockEngineManager.FileInfo] = [] + ) { + self._enabledSources = _enabledSources + self.fileInfos = fileInfos + } + + func set(source: GroupedAdBlockEngine.Source, enabled: Bool) { + if enabled { + _enabledSources.insert(source) + } else { + _enabledSources.remove(source) + } + } + + var enabledSources: [GroupedAdBlockEngine.Source] { + return fileInfos.compactMap { fileInfo in + guard _enabledSources.contains(fileInfo.filterListInfo.source) else { return nil } + return fileInfo.filterListInfo.source + } + } + + func legacyCacheFiles( + for engineType: Brave.GroupedAdBlockEngine.EngineType + ) -> [Brave.AdBlockEngineManager.FileInfo] { + return [] + } +} diff --git a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift index 0c3a40553b2c..88f75be0f6a3 100644 --- a/ios/brave-ios/Tests/ClientTests/PageDataTests.swift +++ b/ios/brave-ios/Tests/ClientTests/PageDataTests.swift @@ -10,19 +10,40 @@ import XCTest @testable import Brave final class PageDataTests: XCTestCase { + @MainActor private lazy var ruleStore: WKContentRuleListStore = { + let testBundle = Bundle.module + let bundleURL = testBundle.bundleURL + return WKContentRuleListStore(url: bundleURL)! + }() + @MainActor func testBasicExample() throws { // Given // Page data with empty ad-block stats let mainFrameURL = URL(string: "http://example.com")! let subFrameURL = URL(string: "http://example.com/1p/subframe")! let upgradedMainFrameURL = URL(string: "https://example.com")! - var pageData = PageData(mainFrameURL: mainFrameURL, adBlockStats: AdBlockStats()) + let sourceProvider = TestSourceProvider() + let groupsManager = AdBlockGroupsManager( + standardManager: AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_standard" + ), + aggressiveManager: AdBlockEngineManager( + engineType: .standard, + cacheFolderName: "test_aggressive" + ), + contentBlockerManager: makeContentBlockingManager(), + sourceProvider: sourceProvider + ) + + var pageData = PageData(mainFrameURL: mainFrameURL, groupsManager: groupsManager) let expectation = expectation(description: "") Task { @MainActor in // When // We get the script types for the main frame let domain = pageData.domain(persistent: false) + let mainFrameRequestTypes = await pageData.makeUserScriptTypes( domain: domain, isDeAmpEnabled: false @@ -103,4 +124,8 @@ final class PageDataTests: XCTestCase { waitForExpectations(timeout: 10) } + + @MainActor private func makeContentBlockingManager() -> ContentBlockerManager { + return ContentBlockerManager(ruleStore: ruleStore) + } } diff --git a/ios/brave-ios/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift similarity index 65% rename from ios/brave-ios/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift rename to ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift index 0592bc26a78c..22ff85f37bfa 100644 --- a/ios/brave-ios/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift +++ b/ios/brave-ios/Tests/ClientTests/Web Filters/GroupedAdBlockEngineTests.swift @@ -9,7 +9,7 @@ import XCTest @testable import Brave -final class CachedAdBlockEngineTests: XCTestCase { +final class GroupedAdBlockEngineTests: XCTestCase { func test3rdPartyCheck() throws { let engine = try AdblockEngine( rules: [ @@ -57,31 +57,25 @@ final class CachedAdBlockEngineTests: XCTestCase { ) } - func testEngineMemoryManagment() throws { + @MainActor func testEngineMemoryManagment() throws { AdblockEngine.setDomainResolver() var engine: AdblockEngine? = AdblockEngine() weak var weakEngine: AdblockEngine? = engine + let localFileURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt" + )! - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, - version: "bundled" - ) - - let filterListInfo = CachedAdBlockEngine.FilterListInfo( + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), - localFileURL: Bundle.module.url( - forResource: "iodkpdagapdfkphljnddpjlldadblomo", - withExtension: "txt" - )!, - version: "bundled", - fileType: .text + version: "bundled" ) - - var cachedEngine: CachedAdBlockEngine? = CachedAdBlockEngine( + + var cachedEngine: GroupedAdBlockEngine? = GroupedAdBlockEngine( engine: engine!, - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: false + group: GroupedAdBlockEngine.FilterListGroup( + infos: [filterListInfo], localFileURL: localFileURL, fileType: .text + ), + type: .standard ) XCTAssertNotNil(cachedEngine) @@ -96,19 +90,19 @@ final class CachedAdBlockEngineTests: XCTestCase { } func testCompilationofResources() throws { - let textFilterListInfo = CachedAdBlockEngine.FilterListInfo( - source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), - localFileURL: Bundle.module.url( - forResource: "iodkpdagapdfkphljnddpjlldadblomo", - withExtension: "txt" - )!, - version: "bundled", - fileType: .text - ) - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, + let localFileURL = Bundle.module.url( + forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt" + )! + let textFilterListInfo = GroupedAdBlockEngine.FilterListInfo( + source: .filterList(componentId: "iodkpdagapdfkphljnddpjlldadblomo", uuid: "default"), version: "bundled" ) + let resourcesInfo = GroupedAdBlockEngine.ResourcesInfo( + localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, version: "bundled" + ) + let group = GroupedAdBlockEngine.FilterListGroup( + infos: [textFilterListInfo], localFileURL: localFileURL, fileType: .text + ) let expectation = expectation(description: "Compiled engine resources") AdblockEngine.setDomainResolver() @@ -120,35 +114,21 @@ final class CachedAdBlockEngineTests: XCTestCase { await filterListInfos.asyncConcurrentForEach { filterListInfo in do { - let engine = try CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: false - ) - - let url = URL(string: "https://stackoverflow.com")! - - let domain = await MainActor.run { - return Domain.getOrCreate(forUrl: url, persistent: false) - } + let engine = try GroupedAdBlockEngine.compile(group: group, type: .standard) + try await engine.useResources(from: resourcesInfo) + let frameURL = URL(string: "https://stackoverflow.com")! let sameDomainTypes = try await engine.makeEngineScriptTypes( - frameURL: url, - isMainFrame: true, - domain: domain, - isDeAmpEnabled: false, - index: 0 + frameURL: frameURL, isMainFrame: true, isDeAmpEnabled: false, index: 0 ) // We should have no scripts injected XCTAssertEqual(sameDomainTypes.count, 0) - if await engine.filterListInfo == textFilterListInfo { + if await engine.group.infos.contains(textFilterListInfo) { // This engine file contains some scriplet rules so we can test this part is working let crossDomainTypes = try await engine.makeEngineScriptTypes( - frameURL: URL(string: "https://reddit.com")!, - isMainFrame: true, - domain: domain, + frameURL: URL(string: "https://reddit.com")!, isMainFrame: true, isDeAmpEnabled: false, index: 0 ) @@ -177,15 +157,7 @@ final class CachedAdBlockEngineTests: XCTestCase { func testPerformance() throws { // Given // Ad block data and an engine manager - let sampleFilterListURL = Bundle.module.url( - forResource: "iodkpdagapdfkphljnddpjlldadblomo", - withExtension: "txt" - )! - let resourcesInfo = CachedAdBlockEngine.ResourcesInfo( - localFileURL: Bundle.module.url(forResource: "resources", withExtension: "json")!, - version: "bundled" - ) - + let sampleFilterListURL = Bundle.module.url(forResource: "iodkpdagapdfkphljnddpjlldadblomo", withExtension: "txt")! // Then // Measure performance let options = XCTMeasureOptions() @@ -194,19 +166,17 @@ final class CachedAdBlockEngineTests: XCTestCase { measure(metrics: [XCTClockMetric(), XCTCPUMetric(), XCTMemoryMetric()], options: options) { let uuid = UUID().uuidString - let filterListInfo = CachedAdBlockEngine.FilterListInfo( + let filterListInfo = GroupedAdBlockEngine.FilterListInfo( source: .filterListURL(uuid: uuid), - localFileURL: sampleFilterListURL, - version: "bundled", - fileType: .text + version: "bundled" ) - + + let group = GroupedAdBlockEngine.FilterListGroup( + infos: [filterListInfo], localFileURL: sampleFilterListURL, fileType: .text + ) + do { - _ = try CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, - resourcesInfo: resourcesInfo, - isAlwaysAggressive: false - ) + _ = try GroupedAdBlockEngine.compile(group: group, type: .standard) } catch { XCTFail(error.localizedDescription) }