Make filter syntax converter process all filters except explicitly ex…

…cluded Also add an advanced option `assetConvertMyFilters` to make the converter process `My Filters` (disabled by default). Related issue: - #228
gorhill · Mar 15, 2020 · 1f23453 · 1f23453
1 parent 4c646ce
commit 1f23453
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 33 deletions.
diff --git a/assets/assets.json b/assets/assets.json
@@ -86,6 +86,7 @@
  "content": "filters",
  "group": "ads",
  "off": true,
+ "noConvert": true,
  "title": "Adblock Warning Removal",
  "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt",
  "supportURL": "https://forums.lanik.us/"
@@ -112,6 +113,7 @@
  "easylist": {
  "content": "filters",
  "group": "ads",
+ "noConvert": true,
  "title": "EasyList",
  "contentURL": [
  "https://easylist.to/easylist/easylist.txt",
@@ -125,6 +127,7 @@
  "content": "filters",
  "group": "privacy",
  "off": true,
+ "noConvert": true,
  "title": "AdGuard Tracking Protection",
  "contentURL": "https://filters.adtidy.org/extension/ublock/filters/3.txt",
  "supportURL": "https://github.com/AdguardTeam/AdguardFilters#adguard-filters",
@@ -133,6 +136,7 @@
  "easyprivacy": {
  "content": "filters",
  "group": "privacy",
+ "noConvert": true,
  "title": "EasyPrivacy",
  "contentURL": [
  "https://easylist.to/easylist/easyprivacy.txt",
@@ -154,12 +158,14 @@
  "content": "filters",
  "group": "malware",
  "off": true,
+ "noConvert": true,
  "title": "Malvertising filter list by Disconnect",
  "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt"
  },
  "malware-0": {
  "content": "filters",
  "group": "malware",
+ "noConvert": true,
  "title": "Malware Domain List",
  "contentURL": [
  "https://gitcdn.xyz/repo/NanoMeow/MDLMirror/master/hosts.txt",
@@ -173,6 +179,7 @@
  "malware-1": {
  "content": "filters",
  "group": "malware",
+ "noConvert": true,
  "title": "Malware domains",
  "contentURL": [
  "https://mirror.cedia.org.ec/malwaredomains/justdomains",
@@ -186,6 +193,7 @@
  "content": "filters",
  "group": "malware",
  "off": true,
+ "noConvert": true,
  "title": "Spam404",
  "contentURL": "https://raw.githubusercontent.com/Spam404/lists/master/adblock-list.txt",
  "supportURL": "http://www.spam404.com/"
@@ -203,6 +211,7 @@
  "content": "filters",
  "group": "social",
  "off": true,
+ "noConvert": true,
  "title": "AdGuard Social Media",
  "contentURL": "https://filters.adtidy.org/extension/ublock/filters/4.txt",
  "supportURL": "https://github.com/AdguardTeam/AdguardFilters#adguard-filters",
@@ -212,6 +221,7 @@
  "content": "filters",
  "group": "social",
  "off": true,
+ "noConvert": true,
  "title": "Anti-Facebook",
  "contentURL": "https://fanboy.co.nz/fanboy-antifacebook.txt",
  "supportURL": "https://github.com/ryanbr/fanboy-adblock/issues"
@@ -220,6 +230,7 @@
  "content": "filters",
  "group": "social",
  "off": true,
+ "noConvert": true,
  "title": "Fanboy’s Annoyance",
  "contentURL": [
  "https://easylist.to/easylist/fanboy-annoyance.txt",
@@ -231,6 +242,7 @@
  "content": "filters",
  "group": "social",
  "off": true,
+ "noConvert": true,
  "title": "EasyList Cookie",
  "contentURL": "https://easylist-downloads.adblockplus.org/easylist-cookie.txt",
  "supportURL": "https://github.com/easylist/easylist/issues"
@@ -239,6 +251,7 @@
  "content": "filters",
  "group": "social",
  "off": true,
+ "noConvert": true,
  "title": "Fanboy’s Social",
  "contentURL": [
  "https://easylist.to/easylist/fanboy-social.txt",
@@ -258,6 +271,7 @@
  "group": "multipurpose",
  "updateAfter": 11,
  "off": true,
+ "noConvert": true,
  "title": "Dan Pollock’s hosts file",
  "contentURL": "https://someonewhocares.org/hosts/hosts",
  "supportURL": "https://someonewhocares.org/hosts/"
@@ -267,6 +281,7 @@
  "group": "multipurpose",
  "updateAfter": 11,
  "off": true,
+ "noConvert": true,
  "title": "hpHosts’ Ad and tracking servers",
  "contentURL": [
  "https://hosts-file.net/.%5Cad_servers.txt",
@@ -279,6 +294,7 @@
  "group": "multipurpose",
  "updateAfter": 11,
  "off": true,
+ "noConvert": true,
  "title": "MVPS HOSTS",
  "contentURL": "http://winhelp2002.mvps.org/hosts.txt",
  "supportURL": "http://winhelp2002.mvps.org/"
@@ -287,6 +303,7 @@
  "content": "filters",
  "group": "multipurpose",
  "updateAfter": 13,
+ "noConvert": true,
  "title": "Peter Lowe’s Ad and tracking server list",
  "contentURL": [
  "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext",

diff --git a/src/js/assets.js b/src/js/assets.js
@@ -171,7 +171,7 @@ api.fetchText = function(url, onLoad, onError) {
 // https://github.com/gorhill/uBlock/issues/3331
 // Support the seamless loading of sublists.
 
-api.fetchFilterList = function(mainlistURL, onLoad, onError) {
+api.fetchFilterList = function(mainlistURL, convert, onLoad, onError) {
  var content = [],
  errored = false,
  pendingSublistURLs = new Set([ mainlistURL ]),
@@ -232,7 +232,7 @@ api.fetchFilterList = function(mainlistURL, onLoad, onError) {
  details.url = mainlistURL;
  details.content = content.join('\n').trim();
 
- if (api.fetchFilterList.legacy.regexFilters.test(mainlistURL)) {
+ if ( convert ) {
  details.content = api.fetchFilterList.legacy.convert(details.content);
  }
 
@@ -263,34 +263,6 @@ api.fetchFilterList.toParsedURL = function(url) {
 };
 
 api.fetchFilterList.legacy = {
- mapFilters: [
- // uBlock filters
- '^assets/ublock/',
- 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/',
- // Fanboy’s Enhanced Tracking List
- 'https://www.fanboy.co.nz/enhancedstats.txt',
- // EST: Eesti saitidele kohandatud filter
- 'https://adblock.ee/list.php',
- // HUN: hufilter
- 'https://raw.githubusercontent.com/hufilter/hufilter/master/hufilter.txt',
- // IDN, MYS: ABPindo
- 'https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt',
- // IRN: Adblock-Iran
- 'https://gitcdn.xyz/repo/farrokhi/adblock-iran/master/filter.txt',
- // NOR, DNK, ISL: Dandelion Sprouts nordiske filtre
- 'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/NorwegianList.txt',
- 'https://repo.or.cz/FilterMirrorRepo.git/blob_plain/refs/heads/master:/NorwegianList.txt',
- // POL: Oficjalne polskie filtry przeciwko alertom o Adblocku
- 'https://raw.githubusercontent.com/olegwukr/polish-privacy-filters/master/anti-adblock.txt',
- // RUS: RU AdList + Counters
- 'https://easylist-downloads.adblockplus.org/advblock+cssfixes.txt',
- 'https://easylist-downloads.adblockplus.org/cntblock.txt'
- ],
- get regexFilters() {
- delete this.regexFilters;
- return this.regexFilters = new RegExp(this.mapFilters
- .join('|').replace(/[\\\/.+]/g, '\\$&'));
- },
  mapRules: {
  '=1x1.gif': '=1x1-transparent.gif',
  '=2x2.png': '=2x2-transparent.png',
@@ -882,7 +854,8 @@ api.get = function(assetKey, options, callback) {
  return reportBack('', 'E_NOTFOUND');
  }
  if ( assetDetails.content === 'filters' ) {
- api.fetchFilterList(contentURL, onContentLoaded, onContentNotLoaded);
+ api.fetchFilterList(contentURL, !assetDetails.noConvert,
+ onContentLoaded, onContentNotLoaded);
  } else {
  api.fetchText(contentURL, onContentLoaded, onContentNotLoaded);
  }
@@ -970,7 +943,8 @@ var getRemote = function(assetKey, callback) {
  return reportBack('', 'E_NOTFOUND');
  }
  if ( assetDetails.content === 'filters' ) {
- api.fetchFilterList(contentURL, onRemoteContentLoaded, onRemoteContentError);
+ api.fetchFilterList(contentURL, !assetDetails.noConvert,
+ onRemoteContentLoaded, onRemoteContentError);
  } else {
  api.fetchText(contentURL, onRemoteContentLoaded, onRemoteContentError);
  }
@@ -992,6 +966,12 @@ var getRemote = function(assetKey, callback) {
 /******************************************************************************/
 
 api.put = function(assetKey, content, callback) {
+ if (
+ µBlock.hiddenSettings.assetConvertMyFilters &&
+ assetKey === µBlock.userFiltersPath
+ ) {
+ content = api.fetchFilterList.legacy.convert(content);
+ }
  if ( reIsUserAsset.test(assetKey) ) {
  return saveUserAsset(assetKey, content, callback);
  }

diff --git a/src/js/background.js b/src/js/background.js
@@ -41,6 +41,7 @@ var µBlock = (function() { // jshint ignore:line
  oneMinute = 60 * oneSecond;
 
  var hiddenSettingsDefault = {
+ assetConvertMyFilters: false,
  assetFetchTimeout: 30,
  autoUpdateAssetFetchPeriod: 120,
  autoUpdatePeriod: 7,

diff --git a/src/js/start.js b/src/js/start.js
@@ -140,7 +140,7 @@ var onVersionReady = function(lastVersion) {
 
  // Force updating all filters due to filter syntax converter update.
  // https://github.com/gorhill/uBlock-for-firefox-legacy/issues/224
- if ( vAPI.app.compareVersions(lastVersion, "1.16.4.18") <= 0 ) {
+ if ( vAPI.app.compareVersions(lastVersion, "1.16.4.19") <= 0 ) {
  µb.assets.purge(/./, 'public_suffix_list.dat');
  }