From 38390bab9cffe3c2a52de88c130c42d6c2de768e Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Thu, 5 Dec 2024 12:56:25 -0500 Subject: [PATCH] [mv3] Add `urlskip` support for strict-blocked page + extra info Add information about which ruleset caused a page to be strict- blocked. Whenever possible, add ability to URL-skip an incoming redirect in a strict-blocked page. Add new default list: "30-day Phishing Domain List" --- .../mv3/extension/_locales/en/messages.json | 8 + platform/mv3/extension/css/settings.css | 2 +- platform/mv3/extension/css/strictblock.css | 12 ++ platform/mv3/extension/js/background.js | 25 +-- platform/mv3/extension/js/config.js | 5 +- platform/mv3/extension/js/ruleset-manager.js | 44 ++++- platform/mv3/extension/js/strictblock.js | 115 ++++++++++--- platform/mv3/make-rulesets.js | 58 ++++++- platform/mv3/strictblock.html | 10 +- src/js/messaging.js | 6 +- src/js/static-dnr-filtering.js | 1 + src/js/static-net-filtering.js | 158 +++--------------- src/js/urlskip.js | 157 +++++++++++++++++ tools/make-mv3.sh | 1 + tools/make-nodejs.sh | 1 + 15 files changed, 418 insertions(+), 185 deletions(-) create mode 100644 src/js/urlskip.js diff --git a/platform/mv3/extension/_locales/en/messages.json b/platform/mv3/extension/_locales/en/messages.json index 17a5b24cd7935..ad9d26911371e 100644 --- a/platform/mv3/extension/_locales/en/messages.json +++ b/platform/mv3/extension/_locales/en/messages.json @@ -250,6 +250,14 @@ "strictblockSentence1": { "message": "uBO Lite has prevented the following page from loading:", "description": "Sentence used in the strict-blocked page" + }, + "strictblockReasonSentence1": { + "message": "The page was blocked because of a matching filter in {{listname}}.", + "description": "Text informing about what is causing the page to be blocked" + }, + "strictblockRedirectSentence1": { + "message": "The blocked page wants to redirect to another site. If you choose to proceed, you will navigate directly to: {{url}}", + "description": "Text warning about an incoming redirect" }, "strictblockNoParamsPrompt": { "message": "without parameters", diff --git a/platform/mv3/extension/css/settings.css b/platform/mv3/extension/css/settings.css index a8b07d2a1894c..02eb600370a5d 100644 --- a/platform/mv3/extension/css/settings.css +++ b/platform/mv3/extension/css/settings.css @@ -15,7 +15,7 @@ h3 { label + legend { color: color-mix(in srgb, currentColor 69%, transparent); - font-size: small; + font-size: var(--font-size-smaller); margin-inline-start: var(--default-gap-large); } diff --git a/platform/mv3/extension/css/strictblock.css b/platform/mv3/extension/css/strictblock.css index f7534078cca7b..7f1f0e9c0d162 100644 --- a/platform/mv3/extension/css/strictblock.css +++ b/platform/mv3/extension/css/strictblock.css @@ -26,6 +26,9 @@ body { :root.mobile body { padding: var(--default-gap-small); } +body.loading { + visibility: hidden; + } #rootContainer { width: min(100%, 640px); @@ -43,6 +46,9 @@ p { a { text-decoration: none; } +q { + font-weight: bold; + } .code { font-size: 13px; word-break: break-all; @@ -128,6 +134,12 @@ body[dir="rtl"] #toggleParse { font-weight: bold; } +#urlskip a { + display: block; + overflow-y: auto; + word-break: break-all; + } + #actionContainer { display: flex; justify-content: space-between; diff --git a/platform/mv3/extension/js/background.js b/platform/mv3/extension/js/background.js index a58dd70639a48..e250d1025ed96 100644 --- a/platform/mv3/extension/js/background.js +++ b/platform/mv3/extension/js/background.js @@ -50,6 +50,7 @@ import { excludeFromStrictBlock, getEnabledRulesetsDetails, getRulesetDetails, + patchDefaultRulesets, setStrictBlockMode, updateDynamicRules, } from './ruleset-manager.js'; @@ -377,20 +378,24 @@ function onMessage(request, sender, callback) { async function start() { await loadRulesetConfig(); + const currentVersion = getCurrentVersion(); + const isNewVersion = currentVersion !== rulesetConfig.version; + + // The default rulesets may have changed, find out new ruleset to enable, + // obsolete ruleset to remove. + if ( isNewVersion ) { + ubolLog(`Version change: ${rulesetConfig.version} => ${currentVersion}`); + rulesetConfig.version = currentVersion; + await patchDefaultRulesets(); + saveRulesetConfig(); + } + const rulesetsUpdated = process.wakeupRun === false && await enableRulesets(rulesetConfig.enabledRulesets); // We need to update the regex rules only when ruleset version changes. - if ( process.wakeupRun === false ) { - const currentVersion = getCurrentVersion(); - if ( currentVersion !== rulesetConfig.version ) { - ubolLog(`Version change: ${rulesetConfig.version} => ${currentVersion}`); - rulesetConfig.version = currentVersion; - saveRulesetConfig(); - if ( rulesetsUpdated === false ) { - updateDynamicRules(); - } - } + if ( isNewVersion && rulesetsUpdated === false ) { + updateDynamicRules(); } // Permissions may have been removed while the extension was disabled diff --git a/platform/mv3/extension/js/config.js b/platform/mv3/extension/js/config.js index e3859e63fa42f..0c79dd62b18f0 100644 --- a/platform/mv3/extension/js/config.js +++ b/platform/mv3/extension/js/config.js @@ -24,13 +24,11 @@ import { sessionRead, sessionWrite, } from './ext.js'; -import { defaultRulesetsFromLanguage } from './ruleset-manager.js'; - /******************************************************************************/ export const rulesetConfig = { version: '', - enabledRulesets: [ 'default' ], + enabledRulesets: [], autoReload: true, showBlockedCount: true, strictBlockMode: true, @@ -67,7 +65,6 @@ export async function loadRulesetConfig() { sessionWrite('rulesetConfig', rulesetConfig); return; } - rulesetConfig.enabledRulesets = await defaultRulesetsFromLanguage(); sessionWrite('rulesetConfig', rulesetConfig); localWrite('rulesetConfig', rulesetConfig); process.firstRun = true; diff --git a/platform/mv3/extension/js/ruleset-manager.js b/platform/mv3/extension/js/ruleset-manager.js index c9490e6836592..04bfb3b6f241d 100644 --- a/platform/mv3/extension/js/ruleset-manager.js +++ b/platform/mv3/extension/js/ruleset-manager.js @@ -561,8 +561,6 @@ async function filteringModesToDNR(modes) { /******************************************************************************/ async function defaultRulesetsFromLanguage() { - const out = await dnr.getEnabledRulesets(); - const dropCountry = lang => { const pos = lang.indexOf('-'); if ( pos === -1 ) { return lang; } @@ -581,7 +579,12 @@ async function defaultRulesetsFromLanguage() { ); const rulesetDetails = await getRulesetDetails(); + const out = []; for ( const [ id, details ] of rulesetDetails ) { + if ( details.enabled ) { + out.push(id); + continue; + } if ( typeof details.lang !== 'string' ) { continue; } if ( reTargetLang.test(details.lang) === false ) { continue; } out.push(id); @@ -591,6 +594,42 @@ async function defaultRulesetsFromLanguage() { /******************************************************************************/ +async function patchDefaultRulesets() { + const [ + oldDefaultIds = [], + newDefaultIds, + newIds, + ] = await Promise.all([ + localRead('defaultRulesetIds'), + defaultRulesetsFromLanguage(), + getRulesetDetails(), + ]); + const toAdd = []; + const toRemove = []; + for ( const id of newDefaultIds ) { + if ( oldDefaultIds.includes(id) ) { continue; } + toAdd.push(id); + } + for ( const id of oldDefaultIds ) { + if ( newDefaultIds.includes(id) ) { continue; } + toRemove.push(id); + } + for ( const id of rulesetConfig.enabledRulesets ) { + if ( newIds.has(id) ) { continue; } + toRemove.push(id); + } + localWrite('defaultRulesetIds', newDefaultIds); + if ( toAdd.length === 0 && toRemove.length === 0 ) { return; } + const enabledRulesets = new Set(rulesetConfig.enabledRulesets); + toAdd.forEach(id => enabledRulesets.add(id)); + toRemove.forEach(id => enabledRulesets.delete(id)); + const patchedRulesets = Array.from(enabledRulesets); + ubolLog(`Patched rulesets: ${rulesetConfig.enabledRulesets} => ${patchedRulesets}`); + rulesetConfig.enabledRulesets = patchedRulesets; +} + +/******************************************************************************/ + async function enableRulesets(ids) { const afterIds = new Set(ids); const [ beforeIds, adminIds, rulesetDetails ] = await Promise.all([ @@ -679,6 +718,7 @@ export { filteringModesToDNR, getRulesetDetails, getEnabledRulesetsDetails, + patchDefaultRulesets, setStrictBlockMode, updateDynamicRules, }; diff --git a/platform/mv3/extension/js/strictblock.js b/platform/mv3/extension/js/strictblock.js index 616ef6ddbc32d..7600bcfbb9e7f 100644 --- a/platform/mv3/extension/js/strictblock.js +++ b/platform/mv3/extension/js/strictblock.js @@ -20,21 +20,15 @@ */ import { dom, qs$ } from './dom.js'; +import { fetchJSON } from './fetch.js'; +import { getEnabledRulesetsDetails } from './ruleset-manager.js'; import { i18n$ } from './i18n.js'; import { sendMessage } from './ext.js'; +import { urlSkip } from './urlskip.js'; /******************************************************************************/ -const toURL = new URL('about:blank'); - -function setURL(url) { - try { - toURL.href = url; - } catch(_) { - } -} - -setURL(self.location.hash.slice(1)); +const rulesetDetailsPromise = getEnabledRulesetsDetails(); /******************************************************************************/ @@ -66,6 +60,13 @@ async function proceed() { /******************************************************************************/ +const toURL = new URL('about:blank'); + +try { + toURL.href = self.location.hash.slice(1); +} catch(_) { +} + dom.clear('#theURL > p > span:first-of-type'); qs$('#theURL > p > span:first-of-type').append(urlToFragment(toURL.href)); @@ -152,16 +153,92 @@ qs$('#theURL > p > span:first-of-type').append(urlToFragment(toURL.href)); /******************************************************************************/ +// Find which list caused the blocking. +(async ( ) => { + const rulesetDetails = await rulesetDetailsPromise; + let iList = -1; + const searchInList = async i => { + if ( iList !== -1 ) { return; } + const hostnames = new Set( + await fetchJSON(`/rulesets/strictblock/${rulesetDetails[i].id}`) + ); + if ( iList !== -1 ) { return; } + let hn = toURL.hostname; + for (;;) { + if ( hostnames.has(hn) ) { iList = i; break; } + const pos = hn.indexOf('.'); + if ( pos === -1 ) { break; } + hn = hn.slice(pos+1); + } + }; + const toFetch = []; + for ( let i = 0; i < rulesetDetails.length; i++ ) { + if ( rulesetDetails[i].rules.strictblock === 0 ) { continue; } + toFetch.push(searchInList(i)); + } + if ( toFetch.length === 0 ) { return; } + await Promise.all(toFetch); + if ( iList === -1 ) { return; } + + const fragment = new DocumentFragment(); + const text = i18n$('strictblockReasonSentence1'); + const placeholder = '{{listname}}'; + const pos = text.indexOf(placeholder); + if ( pos === -1 ) { return; } + const q = document.createElement('q'); + q.append(rulesetDetails[iList].name); + fragment.append(text.slice(0, pos), q, text.slice(pos + placeholder.length)); + qs$('#reason').append(fragment); + dom.attr('#reason', 'hidden', null); +})(); + +/******************************************************************************/ + +// Offer to skip redirection whenever possible +(async ( ) => { + const rulesetDetails = await rulesetDetailsPromise; + const toFetch = []; + for ( const details of rulesetDetails ) { + if ( details.rules.urlskip === 0 ) { continue; } + toFetch.push(fetchJSON(`/rulesets/urlskip/${details.id}`)); + } + if ( toFetch.length === 0 ) { return; } + const urlskipLists = await Promise.all(toFetch); + for ( const urlskips of urlskipLists ) { + for ( const urlskip of urlskips ) { + const re = new RegExp(urlskip.re, urlskip.c ? undefined : 'i'); + if ( re.test(toURL.href) === false ) { continue; } + const finalURL = urlSkip(toURL.href, false, urlskip.steps); + if ( finalURL === undefined ) { continue; } + const fragment = new DocumentFragment(); + const text = i18n$('strictblockRedirectSentence1'); + const linkPlaceholder = '{{url}}'; + const pos = text.indexOf(linkPlaceholder); + if ( pos === -1 ) { return; } + const link = document.createElement('a'); + link.href = finalURL; + dom.cl.add(link, 'code'); + link.append(urlToFragment(finalURL)); + fragment.append( + text.slice(0, pos), + link, + text.slice(pos + linkPlaceholder.length) + ); + qs$('#urlskip').append(fragment); + dom.attr('#urlskip', 'hidden', null); + return; + } + } +})(); + +/******************************************************************************/ + // https://www.reddit.com/r/uBlockOrigin/comments/breeux/ if ( window.history.length > 1 ) { - dom.on('#back', 'click', ( ) => { - window.history.back(); - }); + dom.on('#back', 'click', ( ) => { window.history.back(); }); qs$('#bye').style.display = 'none'; } else { - dom.on('#bye', 'click', ( ) => { - window.close(); - }); + dom.on('#bye', 'click', ( ) => { window.close(); }); qs$('#back').style.display = 'none'; } @@ -171,8 +248,8 @@ dom.on('#disableWarning', 'change', ev => { dom.cl.toggle('[data-i18n="strictblockClose"]', 'disabled', checked); }); -dom.on('#proceed', 'click', ( ) => { - proceed(); -}); +dom.on('#proceed', 'click', ( ) => { proceed(); }); + +dom.cl.remove(dom.body, 'loading'); /******************************************************************************/ diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index b570067055473..4ea7ae6dba1d0 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -225,7 +225,7 @@ const isRegex = rule => rule.condition.regexFilter !== undefined; const isRedirect = rule => { - if ( rule.action === undefined ) { return false; } + if ( isUnsupported(rule) ) { return false; } if ( rule.action.type !== 'redirect' ) { return false; } if ( rule.action.redirect?.extensionPath !== undefined ) { return true; } if ( rule.action.redirect?.transform?.path !== undefined ) { return true; } @@ -233,19 +233,26 @@ const isRedirect = rule => { }; const isModifyHeaders = rule => - rule.action !== undefined && + isUnsupported(rule) === false && rule.action.type === 'modifyHeaders'; const isRemoveparam = rule => - rule.action !== undefined && + isUnsupported(rule) === false && rule.action.type === 'redirect' && rule.action.redirect.transform !== undefined; -const isGood = rule => +const isSafe = rule => + isUnsupported(rule) === false && + rule.action !== undefined && ( + rule.action.type === 'block' || + rule.action.type === 'allow' || + rule.action.type === 'allowAllRequests' + ); + +const isURLSkip = rule => isUnsupported(rule) === false && - isRedirect(rule) === false && - isModifyHeaders(rule) === false && - isRemoveparam(rule) === false; + rule.action !== undefined && + rule.action.type === 'urlskip'; /******************************************************************************/ @@ -357,7 +364,7 @@ async function processNetworkFilters(assetDetails, network) { } } - const plainGood = rules.filter(rule => isGood(rule) && isRegex(rule) === false); + const plainGood = rules.filter(rule => isSafe(rule) && isRegex(rule) === false); log(`\tPlain good: ${plainGood.length}`); log(plainGood .filter(rule => Array.isArray(rule._warning)) @@ -365,7 +372,7 @@ async function processNetworkFilters(assetDetails, network) { .join('\n'), true ); - const regexes = rules.filter(rule => isGood(rule) && isRegex(rule)); + const regexes = rules.filter(rule => isSafe(rule) && isRegex(rule)); log(`\tMaybe good (regexes): ${regexes.length}`); const redirects = rules.filter(rule => @@ -394,6 +401,22 @@ async function processNetworkFilters(assetDetails, network) { ); log(`\tmodifyHeaders=: ${modifyHeaders.length}`); + const urlskips = rules.filter(rule => isURLSkip(rule)).filter(rule => + rule.__modifierAction === 0 && + rule.condition && + rule.condition.regexFilter && + rule.condition.resourceTypes && + rule.condition.resourceTypes.includes('main_frame') + ).map(rule => { + const steps = rule.__modifierValue; + return { + re: rule.condition.regexFilter, + c: rule.condition.isUrlFilterCaseSensitive, + steps: steps.includes(' ') && steps.split(/ +/) || [ steps ], + }; + }); + log(`\turlskip=: ${urlskips.length}`); + const bad = rules.filter(rule => isUnsupported(rule) ); @@ -460,6 +483,13 @@ async function processNetworkFilters(assetDetails, network) { ); } + if ( urlskips.length !== 0 ) { + writeFile( + `${rulesetDir}/urlskip/${assetDetails.id}.json`, + JSON.stringify(urlskips, null, 1) + ); + } + return { total: rules.length, plain: plainGood.length, @@ -470,6 +500,7 @@ async function processNetworkFilters(assetDetails, network) { redirect: redirects.length, modifyHeaders: modifyHeaders.length, strictblock: strictBlocked.size, + urlskip: urlskips.length, }; } @@ -1114,6 +1145,7 @@ async function rulesetFromURLs(assetDetails) { redirect: netStats.redirect, modifyHeaders: netStats.modifyHeaders, strictblock: netStats.strictblock, + urlskip: netStats.urlskip, discarded: netStats.discarded, rejected: netStats.rejected, }, @@ -1264,6 +1296,14 @@ async function main() { }); // Handpicked rulesets from abroad + await rulesetFromURLs({ + id: 'nrd.30day.phishing', + name: '30-day Phishing Domain List', + enabled: true, + urls: [ 'https://raw.githubusercontent.com/xRuffKez/NRD/refs/heads/main/lists/30-day_phishing/domains-only/nrd-phishing-30day.txt' ], + homeURL: 'https://github.com/xRuffKez/NRD?tab=readme-ov-file', + }); + await rulesetFromURLs({ id: 'stevenblack-hosts', name: 'Steven Black’s Unified Hosts (adware + malware)', diff --git a/platform/mv3/strictblock.html b/platform/mv3/strictblock.html index 9572eab09715b..8b6ef2d9be94b 100644 --- a/platform/mv3/strictblock.html +++ b/platform/mv3/strictblock.html @@ -10,20 +10,26 @@ - +
-

_

+

+ + + +
diff --git a/src/js/messaging.js b/src/js/messaging.js index 16d527c60b48e..b24a9a5a27560 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -1887,7 +1887,11 @@ const onMessage = function(request, sender, callback) { listPromises.push( io.get(assetKey, { dontCache: true }).then(details => { listNames.push(assetKey); - return { name: assetKey, text: details.content }; + return { + name: assetKey, + text: details.content, + trustedSource: assetKey.startsWith('ublock-'), + }; }) ); } diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index d3e7732a6771f..63639006585c2 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -284,6 +284,7 @@ function addToDNR(context, list) { toDNR: true, nativeCssHas: env.includes('native_css_has'), badTypes: [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE ], + trustedSource: list.trustedSource || undefined, }); const compiler = staticNetFilteringEngine.createCompiler(); diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index f4769c2c21702..f084180ffccf3 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -28,6 +28,7 @@ import BidiTrieContainer from './biditrie.js'; import { CompiledListReader } from './static-filtering-io.js'; import { FilteringContext } from './filtering-context.js'; import HNTrieContainer from './hntrie.js'; +import { urlSkip } from './urlskip.js'; /******************************************************************************/ @@ -4679,6 +4680,25 @@ StaticNetFilteringEngine.prototype.dnrFromCompiled = function(op, context, ...ar dnrAddRuleError(rule, `Incompatible with DNR: uritransform=${rule.__modifierValue}`); break; } + case 'urlskip': { + let urlFilter = rule.condition?.urlFilter; + if ( urlFilter === undefined ) { break; } + let anchor = 0b000; + if ( urlFilter.startsWith('||') ) { + anchor |= 0b100; + urlFilter = urlFilter.slice(2); + } else if ( urlFilter.startsWith('|') ) { + anchor |= 0b10; + urlFilter = urlFilter.slice(1); + } + if ( urlFilter.endsWith('|') ) { + anchor |= 0b001; + urlFilter = urlFilter.slice(0, -1); + } + rule.condition.urlFilter = undefined; + rule.condition.regexFilter = restrFromGenericPattern(urlFilter, anchor); + break; + } default: dnrAddRuleError(rule, `Unsupported modifier ${rule.__modifierType}`); break; @@ -5391,57 +5411,6 @@ StaticNetFilteringEngine.prototype.transformRequest = function(fctxt, out = []) return out; }; -/** - * @trustedOption urlskip - * - * @description - * Extract a URL from another URL according to one or more transformation steps, - * thereby skipping over intermediate network request(s) to remote servers. - * Requires a trusted source. - * - * @param steps - * A serie of space-separated directives representing the transformation steps - * to perform to extract the final URL to which a network request should be - * redirected. - * - * Supported directives: - * - * `?name`: extract the value of parameter `name` as the current string. - * - * `&i`: extract the name of the parameter at position `i` as the current - * string. The position is 1-based. - * - * `/.../`: extract the first capture group of a regex as the current string. - * - * `+https`: prepend the current string with `https://`. - * - * `-base64`: decode the current string as a base64-encoded string. - * - * `-safebase64`: decode the current string as a safe base64-encoded string. - * - * `-uricomponent`: decode the current string as a URI encoded string. - * - * `-blocked`: allow the redirection of blocked requests. By default, blocked - * requests can't by urlskip'ed. - * - * At any given step, the currently extracted string may not necessarily be - * a valid URL, and more transformation steps may be needed to obtain a valid - * URL once all the steps are applied. - * - * An unsupported step or a failed step will abort the transformation and no - * redirection will be performed. - * - * The final step is expected to yield a valid URL. If the result is not a - * valid URL, no redirection will be performed. - * - * @example - * ||example.com/path/to/tracker$urlskip=?url - * ||example.com/path/to/tracker$urlskip=?url ?to - * ||pixiv.net/jump.php?$urlskip=&1 - * ||podtrac.com/pts/redirect.mp3/$urlskip=/\/redirect\.mp3\/(.*?\.mp3\b)/ +https - * - * */ - StaticNetFilteringEngine.prototype.urlSkip = function( fctxt, blocked, @@ -5458,7 +5427,7 @@ StaticNetFilteringEngine.prototype.urlSkip = function( const urlin = fctxt.url; const value = directive.value; const steps = value.includes(' ') && value.split(/ +/) || [ value ]; - const urlout = urlSkip(directive, urlin, blocked, steps); + const urlout = urlSkip(urlin, blocked, steps, directive); if ( urlout === undefined ) { continue; } if ( urlout === urlin ) { continue; } fctxt.redirectURL = urlout; @@ -5469,91 +5438,6 @@ StaticNetFilteringEngine.prototype.urlSkip = function( return out; }; -function urlSkip(directive, url, blocked, steps) { - try { - let redirectBlocked = false; - let urlout = url; - for ( const step of steps ) { - const urlin = urlout; - const c0 = step.charCodeAt(0); - // Extract from URL parameter name at position i - if ( c0 === 0x26 ) { // & - const i = (parseInt(step.slice(1)) || 0) - 1; - if ( i < 0 ) { return; } - const url = new URL(urlin); - if ( i >= url.searchParams.size ) { return; } - const params = Array.from(url.searchParams.keys()); - urlout = decodeURIComponent(params[i]); - continue; - } - // Enforce https - if ( c0 === 0x2B && step === '+https' ) { - const s = urlin.replace(/^https?:\/\//, ''); - if ( /^[\w-]:\/\//.test(s) ) { return; } - urlout = `https://${s}`; - continue; - } - // Decode - if ( c0 === 0x2D ) { - // Base64 - if ( step === '-base64' ) { - urlout = self.atob(urlin); - continue; - } - // Safe Base64 - if ( step === '-safebase64' ) { - urlout = urlin.replace(/[-_]/g, safeBase64Replacer); - urlout = self.atob(urlout); - continue; - } - // URI component - if ( step === '-uricomponent' ) { - urlout = self.decodeURIComponent(urlin); - continue; - } - // Enable skip of blocked requests - if ( step === '-blocked' ) { - redirectBlocked = true; - continue; - } - } - // Regex extraction from first capture group - if ( c0 === 0x2F ) { // / - if ( directive.cache === null ) { - directive.cache = new RegExp(step.slice(1, -1)); - } - const match = directive.cache.exec(urlin); - if ( match === null ) { return; } - if ( match.length <= 1 ) { return; } - urlout = match[1]; - continue; - } - // Extract from URL parameter - if ( c0 === 0x3F ) { // ? - urlout = (new URL(urlin)).searchParams.get(step.slice(1)); - if ( urlout === null ) { return; } - if ( urlout.includes(' ') ) { - urlout = urlout.replace(/ /g, '%20'); - } - continue; - } - // Unknown directive - return; - } - const urlfinal = new URL(urlout); - if ( urlfinal.protocol !== 'https:' ) { - if ( urlfinal.protocol !== 'http:' ) { return; } - urlout = urlout.replace('http', 'https'); - } - if ( blocked && redirectBlocked !== true ) { return; } - return urlout; - } catch(x) { - } -} - -const safeBase64Map = { '-': '+', '_': '/' }; -const safeBase64Replacer = s => safeBase64Map[s]; - /******************************************************************************/ // https://github.com/uBlockOrigin/uBlock-issues/issues/1626 diff --git a/src/js/urlskip.js b/src/js/urlskip.js new file mode 100644 index 0000000000000..e4182bbc4be24 --- /dev/null +++ b/src/js/urlskip.js @@ -0,0 +1,157 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2022-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +const safeBase64Map = { '-': '+', '_': '/' }; +const safeBase64Replacer = s => safeBase64Map[s]; + +/** + * @trustedOption urlskip + * + * @description + * Extract a URL from another URL according to one or more transformation steps, + * thereby skipping over intermediate network request(s) to remote servers. + * Requires a trusted source. + * + * @param steps + * A serie of space-separated directives representing the transformation steps + * to perform to extract the final URL to which a network request should be + * redirected. + * + * Supported directives: + * + * `?name`: extract the value of parameter `name` as the current string. + * + * `&i`: extract the name of the parameter at position `i` as the current + * string. The position is 1-based. + * + * `/.../`: extract the first capture group of a regex as the current string. + * + * `+https`: prepend the current string with `https://`. + * + * `-base64`: decode the current string as a base64-encoded string. + * + * `-safebase64`: decode the current string as a safe base64-encoded string. + * + * `-uricomponent`: decode the current string as a URI encoded string. + * + * `-blocked`: allow the redirection of blocked requests. By default, blocked + * requests can't by urlskip'ed. + * + * At any given step, the currently extracted string may not necessarily be + * a valid URL, and more transformation steps may be needed to obtain a valid + * URL once all the steps are applied. + * + * An unsupported step or a failed step will abort the transformation and no + * redirection will be performed. + * + * The final step is expected to yield a valid URL. If the result is not a + * valid URL, no redirection will be performed. + * + * @example + * ||example.com/path/to/tracker$urlskip=?url + * ||example.com/path/to/tracker$urlskip=?url ?to + * ||pixiv.net/jump.php?$urlskip=&1 + * ||podtrac.com/pts/redirect.mp3/$urlskip=/\/redirect\.mp3\/(.*?\.mp3\b)/ +https + * + * */ + +export function urlSkip(url, blocked, steps, directive = {}) { + try { + let redirectBlocked = false; + let urlout = url; + for ( const step of steps ) { + const urlin = urlout; + const c0 = step.charCodeAt(0); + // Extract from URL parameter name at position i + if ( c0 === 0x26 ) { // & + const i = (parseInt(step.slice(1)) || 0) - 1; + if ( i < 0 ) { return; } + const url = new URL(urlin); + if ( i >= url.searchParams.size ) { return; } + const params = Array.from(url.searchParams.keys()); + urlout = decodeURIComponent(params[i]); + continue; + } + // Enforce https + if ( c0 === 0x2B && step === '+https' ) { + const s = urlin.replace(/^https?:\/\//, ''); + if ( /^[\w-]:\/\//.test(s) ) { return; } + urlout = `https://${s}`; + continue; + } + // Decode + if ( c0 === 0x2D ) { + // Base64 + if ( step === '-base64' ) { + urlout = self.atob(urlin); + continue; + } + // Safe Base64 + if ( step === '-safebase64' ) { + urlout = urlin.replace(/[-_]/g, safeBase64Replacer); + urlout = self.atob(urlout); + continue; + } + // URI component + if ( step === '-uricomponent' ) { + urlout = self.decodeURIComponent(urlin); + continue; + } + // Enable skip of blocked requests + if ( step === '-blocked' ) { + redirectBlocked = true; + continue; + } + } + // Regex extraction from first capture group + if ( c0 === 0x2F ) { // / + const re = directive.cache ?? new RegExp(step.slice(1, -1)); + if ( directive.cache === null ) { + directive.cache = re; + } + const match = re.exec(urlin); + if ( match === null ) { return; } + if ( match.length <= 1 ) { return; } + urlout = match[1]; + continue; + } + // Extract from URL parameter + if ( c0 === 0x3F ) { // ? + urlout = (new URL(urlin)).searchParams.get(step.slice(1)); + if ( urlout === null ) { return; } + if ( urlout.includes(' ') ) { + urlout = urlout.replace(/ /g, '%20'); + } + continue; + } + // Unknown directive + return; + } + const urlfinal = new URL(urlout); + if ( urlfinal.protocol !== 'https:' ) { + if ( urlfinal.protocol !== 'http:' ) { return; } + urlout = urlout.replace('http', 'https'); + } + if ( blocked && redirectBlocked !== true ) { return; } + return urlout; + } catch(x) { + } +} diff --git a/tools/make-mv3.sh b/tools/make-mv3.sh index 287a7b08b26bd..5c812ec214dd5 100755 --- a/tools/make-mv3.sh +++ b/tools/make-mv3.sh @@ -76,6 +76,7 @@ cp "$UBO_DIR"/src/css/fa-icons.css "$DES"/css/ cp "$UBO_DIR"/src/js/dom.js "$DES"/js/ cp "$UBO_DIR"/src/js/fa-icons.js "$DES"/js/ cp "$UBO_DIR"/src/js/i18n.js "$DES"/js/ +cp "$UBO_DIR"/src/js/urlskip.js "$DES"/js/ cp "$UBO_DIR"/src/lib/punycode.js "$DES"/js/ cp -R "$UBO_DIR/src/img/flags-of-the-world" "$DES"/img diff --git a/tools/make-nodejs.sh b/tools/make-nodejs.sh index 87e96ddade909..ed8db6cbbf77a 100755 --- a/tools/make-nodejs.sh +++ b/tools/make-nodejs.sh @@ -22,6 +22,7 @@ cp src/js/static-net-filtering.js $DES/js cp src/js/static-filtering-io.js $DES/js cp src/js/tasks.js $DES/js cp src/js/text-utils.js $DES/js +cp src/js/urlskip.js $DES/js cp src/js/uri-utils.js $DES/js cp src/js/url-net-filtering.js $DES/js