Harden processing of changes in compiled list format

Related issue:
- uBlockOrigin/uBlock-issues#1365

This commit adds the compiled magic version number to the
compiled data itself, and consequently this allows uBO
to no longer require that any given compiled list with a
mismatched format to be detected and discarded at launch
time.

Given this change, uBO no longer needs to rely on the
deletion of cached data at launch time to ensure it
won't use no longer valid compiled lists.

src/js/background.js

@@ -154,6 +154,13 @@ const µBlock = (( ) => { // jshint ignore:line
         compiledFormatChanged: false,
         selfieIsInvalid: false,
 
+        compiledNetworkSection: 100,
+        compiledCosmeticSection: 200,
+        compiledScriptletSection: 300,
+        compiledHTMLSection: 400,
+        compiledSentinelSection: 1000,
+        compiledBadSubsection: 1,
+
         restoreBackupSettings: {
             lastRestoreFile: '',
             lastRestoreTime: 0,

src/js/cosmetic-filtering.js

@@ -340,8 +340,7 @@ FilterContainer.prototype.keyFromSelector = function(selector) {
 /******************************************************************************/
 
 FilterContainer.prototype.compile = function(parser, writer) {
-    // 1000 = cosmetic filtering
-    writer.select(1000);
+    writer.select(µb.compiledCosmeticSection);
 
     if ( parser.hasOptions() === false ) {
         this.compileGenericSelector(parser, writer);
@@ -551,8 +550,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader, options) {
         return;
     }
 
-    // 1000 = cosmetic filtering
-    reader.select(1000);
+    reader.select(µb.compiledCosmeticSection);
 
     let db, bucket;
 
@@ -643,8 +641,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader, options) {
 /******************************************************************************/
 
 FilterContainer.prototype.skipGenericCompiledContent = function(reader) {
-    // 1000 = cosmetic filtering
-    reader.select(1000);
+    reader.select(µb.compiledCosmeticSection);
 
     while ( reader.next() ) {
         this.acceptedCount += 1;
@@ -685,8 +682,7 @@ FilterContainer.prototype.skipGenericCompiledContent = function(reader) {
 /******************************************************************************/
 
 FilterContainer.prototype.skipCompiledContent = function(reader) {
-    // 1000 = cosmetic filtering
-    reader.select(1000);
+    reader.select(µb.compiledCosmeticSection);
 
     while ( reader.next() ) {
         this.acceptedCount += 1;

src/js/html-filtering.js

@@ -304,8 +304,7 @@
             return;
         }
 
-        // 1002 = html filtering
-        writer.select(1002);
+        writer.select(µb.compiledHTMLSection);
 
         // TODO: Mind negated hostnames, they are currently discarded.
 
@@ -327,8 +326,7 @@
         // Don't bother loading filters if stream filtering is not supported.
         if ( µb.canFilterResponseData === false ) { return; }
 
-        // 1002 = html filtering
-        reader.select(1002);
+        reader.select(µb.compiledHTMLSection);
 
         while ( reader.next() ) {
             acceptedCount += 1;

src/js/reverselookup.js

@@ -64,7 +64,7 @@ if (
         for ( const assetKey in listEntries ) {
             const entry = listEntries[assetKey];
             if ( entry === undefined ) { continue; }
-            const content = extractBlocks(entry.content, 0, 1);
+            const content = extractBlocks(entry.content, 100, 101);
             let pos = 0;
             for (;;) {
                 pos = content.indexOf(compiledFilter, pos);
@@ -165,7 +165,7 @@ if (
         for ( const assetKey in listEntries ) {
             const entry = listEntries[assetKey];
             if ( entry === undefined ) { continue; }
-            let content = extractBlocks(entry.content, 1000, 2000),
+            let content = extractBlocks(entry.content, 200, 1000),
                 isProcedural,
                 found;
             let pos = 0;

src/js/scriptlet-filtering.js

@@ -230,8 +230,7 @@
     };
 
     api.compile = function(parser, writer) {
-        // 1001 = scriptlet injection
-        writer.select(1001);
+        writer.select(µb.compiledScriptletSection);
 
         // Only exception filters are allowed to be global.
         const { raw, exception } = parser.result;
@@ -270,8 +269,7 @@
     //     4                -1
 
     api.fromCompiledContent = function(reader) {
-        // 1001 = scriptlet injection
-        reader.select(1001);
+        reader.select(µb.compiledScriptletSection);
 
         while ( reader.next() ) {
             acceptedCount += 1;

src/js/start.js

@@ -190,7 +190,6 @@ const onUserSettingsReady = function(fetched) {
 
 const onCacheSettingsReady = async function(fetched) {
     if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) {
-        await µb.assets.remove(/^compiled\//);
         µb.compiledFormatChanged = true;
         µb.selfieIsInvalid = true;
     }
@@ -302,10 +301,9 @@ try {
         }),
         µb.cacheStorage.get(
             { compiledMagic: 0, selfieMagic: 0 }
-        ).then(fetched =>
-            onCacheSettingsReady(fetched)
-        ).then(( ) => {
-            log.info(`Integrity of cached data processed ${Date.now()-vAPI.T0} ms after launch`);
+        ).then(fetched => {
+            log.info(`Cache magic numbers ready ${Date.now()-vAPI.T0} ms after launch`);
+            onCacheSettingsReady(fetched);
         }),
         vAPI.storage.get(createDefaultProps()).then(fetched => {
             log.info(`First fetch ready ${Date.now()-vAPI.T0} ms after launch`);

src/js/static-net-filtering.js

@@ -3639,9 +3639,11 @@ FilterContainer.prototype.compile = function(parser, writer) {
         return false;
     }
 
-    // 0 = network filters
-    // 1 = network filters: bad filters
-    writer.select(parsed.badFilter ? 1 : 0);
+    writer.select(
+        parsed.badFilter
+            ? µb.compiledNetworkSection + µb.compiledBadSubsection
+            : µb.compiledNetworkSection
+    );
 
     // Reminder:
     //   `redirect=` is a combination of a `redirect-rule` filter and a
@@ -3808,8 +3810,7 @@ FilterContainer.prototype.compileToAtomicFilter = function(
 /******************************************************************************/
 
 FilterContainer.prototype.fromCompiledContent = function(reader) {
-    // 0 = network filters
-    reader.select(0);
+    reader.select(µb.compiledNetworkSection);
     while ( reader.next() ) {
         this.acceptedCount += 1;
         if ( this.goodFilters.has(reader.line) ) {
@@ -3819,8 +3820,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader) {
         }
     }
 
-    // 1 = network filters: bad filter directives
-    reader.select(1);
+    reader.select(µb.compiledNetworkSection + µb.compiledBadSubsection);
     while ( reader.next() ) {
         this.badFilters.add(reader.line);
     }

src/js/storage.js

@@ -733,12 +733,18 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
 µBlock.getCompiledFilterList = async function(assetKey) {
     const compiledPath = 'compiled/' + assetKey;
 
+    // https://github.com/uBlockOrigin/uBlock-issues/issues/1365
+    //   Verify that the list version matches that of the current compiled
+    //   format.
     if (
         this.compiledFormatChanged === false &&
         this.badLists.has(assetKey) === false
     ) {
         const compiledDetails = await this.assets.get(compiledPath);
-        if ( compiledDetails.content !== '' ) {
+        if (
+            parseInt(compiledDetails.content, 10) ===
+            this.systemSettings.compiledMagic
+        ) {
             compiledDetails.assetKey = assetKey;
             return compiledDetails;
         }
@@ -878,7 +884,13 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
         staticNetFilteringEngine.compile(parser, writer);
     }
 
-    return writer.toString();
+    // https://github.com/uBlockOrigin/uBlock-issues/issues/1365
+    //   Embed version into compiled list itself: it is encoded in as the
+    //   first digits followed by a whitespace.
+    const compiledContent
+        = `${this.systemSettings.compiledMagic}\n` + writer.toString();
+
+    return compiledContent;
 };
 
 /******************************************************************************/
@@ -889,7 +901,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
 
 µBlock.applyCompiledFilters = function(rawText, firstparty) {
     if ( rawText === '' ) { return; }
-    let reader = new this.CompiledLineIO.Reader(rawText);
+    const reader = new this.CompiledLineIO.Reader(rawText);
     this.staticNetFilteringEngine.fromCompiledContent(reader);
     this.staticExtFilteringEngine.fromCompiledContent(reader, {
         skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters,

src/js/utils.js

@@ -153,6 +153,7 @@
             if ( this.block === undefined ) {
                 this.blocks.set(blockId, (this.block = []));
             }
+            return this;
         }
         toString() {
             let result = [];
@@ -179,7 +180,7 @@
             this.blocks = new Map();
             this.properties = new Map();
             let reBlockStart = new RegExp(
-                '^' + this.io.blockStartPrefix + '(\\d+)\\n',
+                `^${this.io.blockStartPrefix}(\\d+)\\n`,
                 'gm'
             );
             let match = reBlockStart.exec(raw);