Fine tune hostname uncloaking through CNAME-lookup

Related issue:
- uBlockOrigin/uBlock-issues#780

Related commit:
- 3a564c199260

This adds two new advanced settings:

- cnameIgnoreRootDocument
  - Default to `true`
  - Tells uBO to skip CNAME-lookup for root document.

- cnameReplayFullURL
  - Default to `false`
  - Tells uBO whether to replay the whole URL or just
    the origin part of it.
    Replaying only the origin part is meant to lower
    undue breakage and improve performance by avoiding
    repeating the pattern-matching of the whole URL --
    which pattern-matching was most likely already
    accomplished with the original request.

This commit is meant to explore enabling CNAME-lookup
by default for the next stable release while:

- Eliminating a development burden by removing the
  need to create a new filtering syntax to deal with
  undesirable CNAME-cloaked hostnames

- Eliminating a filter list maintainer burden by
  removing the need to find/deal with all base
  domains which engage in undesirable CNAME-cloaked
  hostnames

The hope is that the approach implemented in this
commit should require at most a few unbreak rules
with no further need for special filtering syntax
or filter list maintance efforts.

platform/chromium/vapi-background.js

@@ -1259,18 +1259,29 @@ vAPI.Net = class {
             console.info('No requests found to benchmark');
             return;
         }
+        const mappedTypes = new Map([
+            [ 'document', 'main_frame' ],
+            [ 'subdocument', 'sub_frame' ],
+        ]);
         console.info('vAPI.net.onBeforeSuspendableRequest()...');
         const t0 = self.performance.now();
         const promises = [];
+        const details = {
+            documentUrl: '',
+            tabId: -1,
+            parentFrameId: -1,
+            frameId: 0,
+            type: '',
+            url: '',
+        };
         for ( const request of requests ) {
-            const details = {
-                documentUrl: request.frameUrl,
-                tabId: Number.MAX_SAFE_INTEGER,
-                parentFrameId: -1,
-                frameId: 0,
-                type: request.cpt,
-                url: request.url,
-            };
+            details.documentUrl = request.frameUrl;
+            details.tabId = -1;
+            details.parentFrameId = -1;
+            details.frameId = 0;
+            details.type = mappedTypes.get(request.cpt) || request.cpt;
+            details.url = request.url;
+            if ( details.type === 'main_frame' ) { continue; }
             promises.push(this.onBeforeSuspendableRequest(details));
         }
         return Promise.all(promises).then(results => {

platform/firefox/vapi-webrequest.js

@@ -63,16 +63,20 @@
             this.cnames = new Map([ [ '', '' ] ]);
             this.cnameAliasList = null;
             this.cnameIgnoreList = null;
-            this.url = new URL(vAPI.getURL('/'));
+            this.cnameIgnore1stParty = true;
+            this.cnameIgnoreRootDocument = true;
             this.cnameMaxTTL = 60;
+            this.cnameReplayFullURL = false;
             this.cnameTimer = undefined;
         }
         setOptions(options) {
             super.setOptions(options);
             this.cnameAliasList = this.regexFromStrList(options.cnameAliasList);
             this.cnameIgnoreList = this.regexFromStrList(options.cnameIgnoreList);
-            this.cnameIgnore1stParty = options.cnameIgnore1stParty === true;
+            this.cnameIgnore1stParty = options.cnameIgnore1stParty !== false;
+            this.cnameIgnoreRootDocument = options.cnameIgnoreRootDocument !== false;
             this.cnameMaxTTL = options.cnameMaxTTL || 120;
+            this.cnameReplayFullURL = options.cnameReplayFullURL === true;
             this.cnames.clear(); this.cnames.set('', '');
         }
         normalizeDetails(details) {
@@ -123,11 +127,22 @@
             }
             return Array.from(out);
         }
-        processCanonicalName(cname, details) {
-            this.url.href = details.url;
-            details.cnameOf = this.url.hostname;
-            this.url.hostname = cname;
-            details.url = this.url.href;
+        processCanonicalName(hn, cn, details) {
+            const hnBeg = details.url.indexOf(hn);
+            if ( hnBeg === -1 ) { return; }
+            const oldURL = details.url;
+            let newURL = oldURL.slice(0, hnBeg) + cn;
+            const hnEnd = hnBeg + hn.length;
+            if ( this.cnameReplayFullURL ) {
+                newURL += oldURL.slice(hnEnd);
+            } else {
+                const pathBeg = oldURL.indexOf('/', hnEnd);
+                if ( pathBeg !== -1 ) {
+                    newURL += oldURL.slice(hnEnd, pathBeg + 1);
+                }
+            }
+            details.url = newURL;
+            details.aliasURL = oldURL;
             return super.onBeforeSuspendableRequest(details);
         }
         recordCanonicalName(hn, record) {
@@ -187,11 +202,14 @@
             let r = super.onBeforeSuspendableRequest(details);
             if ( r !== undefined ) { return r; }
             if ( this.cnameAliasList === null ) { return; }
+            if ( details.type === 'main_frame' && this.cnameIgnoreRootDocument ) {
+                return;
+            }
             const hn = vAPI.hostnameFromNetworkURL(details.url);
             let cname = this.cnames.get(hn);
             if ( cname === '' ) { return; }
             if ( cname !== undefined ) {
-                return this.processCanonicalName(cname, details);
+                return this.processCanonicalName(hn, cname, details);
             }
             if ( this.cnameAliasList.test(hn) === false ) {
                 this.cnames.set(hn, '');
@@ -201,7 +219,7 @@
                 rec => {
                     const cname = this.recordCanonicalName(hn, rec);
                     if ( cname === '' ) { return; }
-                    return this.processCanonicalName(cname, details);
+                    return this.processCanonicalName(hn, cname, details);
                 },
                 ( ) => {
                     this.cnames.set(hn, '');

src/css/logger-ui.css

@@ -268,7 +268,7 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm,
 body.colorBlind #vwRenderer .logEntry > div.redirect {
     background-color: rgba(0, 19, 110, 0.1);
     }
-#vwRenderer .logEntry > div[data-cnameof] {
+#vwRenderer .logEntry > div[data-aliasid] {
     color: mediumblue;
     }
 #vwRenderer .logEntry > div[data-type="tabLoad"] {

src/js/background.js

@@ -49,7 +49,9 @@ const µBlock = (( ) => { // jshint ignore:line
         cnameAliasList: 'unset',
         cnameIgnoreList: 'unset',
         cnameIgnore1stParty: true,
+        cnameIgnoreRootDocument: true,
         cnameMaxTTL: 120,
+        cnameReplayFullURL: false,
         consoleLogLevel: 'unset',
         debugScriptlets: false,
         debugScriptletInjector: false,

src/js/filtering-context.js

@@ -29,9 +29,10 @@
     }
     this.tstamp = 0;
     this.realm = '';
+    this.id = undefined;
     this.type = undefined;
-    this.cnameOf = undefined;
     this.url = undefined;
+    this.aliasURL = undefined;
     this.hostname = undefined;
     this.domain = undefined;
     this.docId = undefined;
@@ -64,9 +65,10 @@
         }
         this.fromTabId(tabId);
         this.realm = '';
+        this.id = details.requestId;
         this.type = details.type;
         this.setURL(details.url);
-        this.cnameOf = details.cnameOf || undefined;
+        this.aliasURL = details.aliasURL || undefined;
         this.docId = details.type !== 'sub_frame'
             ? details.frameId
             : details.parentFrameId;

src/js/logger-ui.js

@@ -157,9 +157,12 @@ const regexFromURLFilteringResult = function(result) {
 
 const nodeFromURL = function(parent, url, re) {
     const fragment = document.createDocumentFragment();
-    if ( re instanceof RegExp === false ) {
+    if ( re === undefined ) {
         fragment.textContent = url;
     } else {
+        if ( typeof re === 'string' ) {
+            re = new RegExp(re.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g');
+        }
         const matches = re.exec(url);
         if ( matches === null || matches[0].length === 0 ) {
             fragment.textContent = url;
@@ -211,6 +214,9 @@ const LogEntry = function(details) {
             this[prop] = details[prop];
         }
     }
+    if ( details.aliasURL !== undefined ) {
+        this.aliased = true;
+    }
     if ( this.tabDomain === '' ) {
         this.tabDomain = this.tabHostname || '';
     }
@@ -222,12 +228,13 @@ const LogEntry = function(details) {
     }
 };
 LogEntry.prototype = {
-    cnameOf: '',
+    aliased: false,
     dead: false,
     docDomain: '',
     docHostname: '',
     domain: '',
     filter: undefined,
+    id: '',
     realm: '',
     tabDomain: '',
     tabHostname: '',
@@ -294,7 +301,7 @@ const processLoggerEntries = function(response) {
             if ( autoDeleteVoidedRows ) { continue; }
             parsed.voided = true;
         }
-        if ( parsed.type === 'main_frame' && parsed.cnameOf === '' ) {
+        if ( parsed.type === 'main_frame' && parsed.aliased === false ) {
             const separator = createLogSeparator(parsed, unboxed.url);
             loggerEntries.unshift(separator);
             if ( rowFilterer.filterOne(separator) ) {
@@ -304,7 +311,7 @@ const processLoggerEntries = function(response) {
                 }
             }
         }
-        if ( cnameOfEnabled === false && parsed.cnameOf !== '' ) {
+        if ( cnameOfEnabled === false && parsed.aliased ) {
             uDom.nodeFromId('filterExprCnameOf').style.display = '';
             cnameOfEnabled = true;
         }
@@ -405,8 +412,10 @@ const parseLogEntry = function(details) {
     textContent.push(normalizeToStr(details.url));
 
     // Hidden cells -- useful for row-filtering purpose
-    if ( entry.cnameOf !== '' ) {
-        textContent.push(`cnameOf=${entry.cnameOf}`);
+
+    // Cell 7
+    if ( entry.aliased ) {
+        textContent.push(`aliasURL=${details.aliasURL}`);
     }
 
     entry.textContent = textContent.join('\t');
@@ -723,17 +732,20 @@ const viewPort = (( ) => {
         span.textContent = cells[5];
 
         // URL
-        let re = null;
+        let re;
         if ( filteringType === 'static' ) {
             re = new RegExp(filter.regex, 'gi');
         } else if ( filteringType === 'dynamicUrl' ) {
             re = regexFromURLFilteringResult(filter.rule.join(' '));
         }
         nodeFromURL(div.children[6], cells[6], re);
 
-        // Cname
-        if ( details.cnameOf !== '' ) {
-            div.setAttribute('data-cnameof', details.cnameOf);
+        // Alias URL (CNAME, etc.)
+        if ( cells.length > 7 ) {
+            const pos = details.textContent.lastIndexOf('\taliasURL=');
+            if ( pos !== -1 ) {
+                div.setAttribute('data-aliasid', details.id);
+            }
         }
 
         return div;
@@ -1452,6 +1464,16 @@ const reloadTab = function(ev) {
         return targetRow.children[1].textContent;
     };
 
+    const aliasURLFromID = function(id) {
+        if ( id === '' ) { return ''; }
+        for ( const entry of loggerEntries ) {
+            if ( entry.id !== id || entry.aliased ) { continue; }
+            const fields = entry.textContent.split('\t');
+            return fields[6] || '';
+        }
+        return '';
+    };
+
     const toSummaryPaneFilterNode = async function(receiver, filter) {
         receiver.children[1].textContent = filter;
         if ( filterAuthorMode !== true ) { return; }
@@ -1613,8 +1635,8 @@ const reloadTab = function(ev) {
             rows[6].style.display = 'none';
         }
         // URL
-        text = trch[6].textContent;
-        if ( text !== '' ) {
+        const canonicalURL = trch[6].textContent;
+        if ( canonicalURL !== '' ) {
             const attr = tr.getAttribute('data-status') || '';
             if ( attr !== '' ) {
                 rows[7].setAttribute('data-status', attr);
@@ -1623,12 +1645,17 @@ const reloadTab = function(ev) {
         } else {
             rows[7].style.display = 'none';
         }
-        // CNAME of
-        text = tr.getAttribute('data-cnameof') || '';
-        if ( text !== '' ) {
-            rows[8].children[1].textContent = text;
+        // Alias URL
+        text = tr.getAttribute('data-aliasid');
+        const aliasURL = text ? aliasURLFromID(text) : '';
+        if ( aliasURL !== '' ) {
+            rows[8].children[1].textContent =
+                vAPI.hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' +
+                vAPI.hostnameFromURI(canonicalURL);
+            rows[9].children[1].textContent = aliasURL;
         } else {
             rows[8].style.display = 'none';
+            rows[9].style.display = 'none';
         }
     };
 

src/js/storage.js

@@ -139,7 +139,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
         cnameAliasList: µBlock.hiddenSettings.cnameAliasList,
         cnameIgnoreList: µBlock.hiddenSettings.cnameIgnoreList,
         cnameIgnore1stParty: µBlock.hiddenSettings.cnameIgnore1stParty,
+        cnameIgnoreRootDocument: µBlock.hiddenSettings.cnameIgnoreRootDocument,
         cnameMaxTTL: µBlock.hiddenSettings.cnameMaxTTL,
+        cnameReplayFullURL: µBlock.hiddenSettings.cnameReplayFullURL,
     });
 });
 

src/js/traffic.js

@@ -99,7 +99,7 @@ const onBeforeRequest = function(details) {
         if (
             details.parentFrameId !== -1 &&
             details.type === 'sub_frame' &&
-            details.cnameOf === undefined
+            details.aliasURL === undefined
         ) {
             pageStore.setFrame(details.frameId, details.url);
         }

src/logger-ui.html

@@ -65,7 +65,7 @@
                                 </span>
                             </div>
                             <div><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\t(?:0,)?1\t" data-i18n="loggerRowFiltererBuiltin1p"></span><span data-filtex="\t(?:3(?:,\d)?|0,3)\t" data-i18n="loggerRowFiltererBuiltin3p"></span></div>
-                            <div id="filterExprCnameOf" style="display:none"><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\tcnameOf=.">CNAME</span></div>
+                            <div id="filterExprCnameOf" style="display:none"><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\taliasURL=.">CNAME</span></div>
                         </div>
                     </span>
                 </span>
@@ -121,7 +121,8 @@
                 <div><span data-i18n="loggerEntryDetailsPartyness"></span><span class="prose"></span></div>
                 <div><span data-i18n="loggerEntryDetailsType"></span><span></span></div>
                 <div><span data-i18n="loggerEntryDetailsURL"></span><span></span></div>
-                <div><span >CNAME of</span><span></span></div>
+                <div><span>CNAME</span><span></span></div>
+                <div><span>Original URL</span><span></span></div>
             </div>
             <div class="pane dynamic hide" data-pane="dynamic">
                 <div class="toolbar row">