Skip to content

Commit

Permalink
Ignore assets older than cached version when fetching from CDNs
Browse files Browse the repository at this point in the history 
As discussed with filter list volunteers.

uBlockOrigin/uBlock-discussions#781 (reply in thread)
  • Loading branch information
@gorhill
gorhill committed Oct 17, 2023
1 parent f122ce7 commit 7daf313
Showing 1 changed file with 150 additions and 43 deletions.
193 changes: 150 additions & 43 deletions src/js/assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import logger from './logger.js';
import µb from './background.js';
import { i18n$ } from './i18n.js';
import * as sfp from './static-filtering-parser.js';
import { ubolog } from './console.js';

/******************************************************************************/

Expand All @@ -41,6 +42,56 @@ const assets = {};
// bandwidth of remote servers.
let remoteServerFriendly = false;

const resourceTimeFromXhr = xhr => {
try {
// First lookup timestamp from content
let assetTime = 0;
if ( typeof xhr.response === 'string' ) {
const head = xhr.response.slice(0, 512);
const match = /^! Last modified: (.+)$/m.exec(head);
if ( match ) {
assetTime = (new Date(match[1])).getTime() || 0;
}
}
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
let networkTime = 0;
const age = parseInt(xhr.getResponseHeader('Age'), 10);
if ( isNaN(age) === false ) {
const time = (new Date(xhr.getResponseHeader('Date'))).getTime();
if ( isNaN(time) === false ) {
networkTime = time - age * 1000;
}
}
return Math.max(assetTime, networkTime, 0);
} catch(_) {
}
return 0;
};

const resourceTimeFromParts = (parts, time) => {
const goodParts = parts.filter(part => typeof part === 'object');
return goodParts.reduce((acc, part) =>
((part.resourceTime || 0) > acc ? part.resourceTime : acc),
time
);
};

const resourceIsStale = (networkDetails, cacheDetails) => {
if ( typeof networkDetails.resourceTime !== 'number' ) { return false; }
if ( networkDetails.resourceTime === 0 ) { return false; }
if ( typeof cacheDetails.resourceTime !== 'number' ) { return false; }
if ( cacheDetails.resourceTime === 0 ) { return false; }
if ( networkDetails.resourceTime === cacheDetails.resourceTime ) { return true; }
if ( networkDetails.resourceTime < cacheDetails.resourceTime ) {
ubolog(`Skip ${networkDetails.url}\n\tolder than ${cacheDetails.remoteURL}`);
return true;
}
return false;
};

const stringIsNotEmpty = s => typeof s === 'string' && s !== '';

/******************************************************************************/

const observers = [];
Expand Down Expand Up @@ -109,6 +160,7 @@ assets.fetch = function(url, options = {}) {
return fail(details, `${url}: ${details.statusCode} ${details.statusText}`);
}
details.content = this.response;
details.resourceTime = resourceTimeFromXhr(this);
resolve(details);
};

Expand Down Expand Up @@ -312,19 +364,21 @@ assets.fetchFilterList = async function(mainlistURL) {
];
// Abort processing `include` directives if at least one included sublist
// can't be fetched.
let resourceTime = 0;
do {
allParts = await Promise.all(allParts);
const part = allParts.find(part => {
return typeof part === 'object' && part.error !== undefined;
});
const part = allParts
.find(part => typeof part === 'object' && part.error !== undefined);
if ( part !== undefined ) {
return { url: mainlistURL, content: '', error: part.error };
}
resourceTime = resourceTimeFromParts(allParts, resourceTime);
allParts = processIncludeDirectives(allParts);
} while ( allParts.some(part => typeof part !== 'string') );
// If we reach this point, this means all fetches were successful.
return {
url: mainlistURL,
resourceTime,
content: allParts.length === 1
? allParts[0]
: allParts.join('') + '\n'
Expand All @@ -347,7 +401,7 @@ assets.fetchFilterList = async function(mainlistURL) {
let assetSourceRegistryPromise;
let assetSourceRegistry = Object.create(null);

const getAssetSourceRegistry = function() {
function getAssetSourceRegistry() {
if ( assetSourceRegistryPromise === undefined ) {
assetSourceRegistryPromise = cacheStorage.get(
'assetSourceRegistry'
Expand All @@ -373,9 +427,9 @@ const getAssetSourceRegistry = function() {
}

return assetSourceRegistryPromise;
};
}

const registerAssetSource = function(assetKey, newDict) {
function registerAssetSource(assetKey, newDict) {
const currentDict = assetSourceRegistry[assetKey] || {};
for ( const [ k, v ] of Object.entries(newDict) ) {
if ( v === undefined || v === null ) {
Expand Down Expand Up @@ -409,12 +463,12 @@ const registerAssetSource = function(assetKey, newDict) {
currentDict.submitTime = Date.now(); // To detect stale entries
}
assetSourceRegistry[assetKey] = currentDict;
};
}

const unregisterAssetSource = function(assetKey) {
function unregisterAssetSource(assetKey) {
assetCacheRemove(assetKey);
delete assetSourceRegistry[assetKey];
};
}

const saveAssetSourceRegistry = (( ) => {
const save = ( ) => {
Expand All @@ -431,7 +485,14 @@ const saveAssetSourceRegistry = (( ) => {
};
})();

const updateAssetSourceRegistry = function(json, silent = false) {
async function assetSourceGetDetails(assetKey) {
await getAssetSourceRegistry();
const entry = assetSourceRegistry[assetKey];
if ( entry === undefined ) { return; }
return entry;
}

function updateAssetSourceRegistry(json, silent = false) {
let newDict;
try {
newDict = JSON.parse(json);
Expand Down Expand Up @@ -467,7 +528,7 @@ const updateAssetSourceRegistry = function(json, silent = false) {
registerAssetSource(assetKey, newDict[assetKey]);
}
saveAssetSourceRegistry();
};
}

assets.registerAssetSource = async function(assetKey, details) {
await getAssetSourceRegistry();
Expand All @@ -492,7 +553,7 @@ const assetCacheRegistryStartTime = Date.now();
let assetCacheRegistryPromise;
let assetCacheRegistry = {};

const getAssetCacheRegistry = function() {
function getAssetCacheRegistry() {
if ( assetCacheRegistryPromise === undefined ) {
assetCacheRegistryPromise = cacheStorage.get(
'assetCacheRegistry'
Expand Down Expand Up @@ -522,7 +583,7 @@ const getAssetCacheRegistry = function() {
}

return assetCacheRegistryPromise;
};
}

const saveAssetCacheRegistry = (( ) => {
const save = function() {
Expand All @@ -539,7 +600,7 @@ const saveAssetCacheRegistry = (( ) => {
};
})();

const assetCacheRead = async function(assetKey, updateReadTime = false) {
async function assetCacheRead(assetKey, updateReadTime = false) {
const t0 = Date.now();
const internalKey = `cache/${assetKey}`;

Expand Down Expand Up @@ -580,9 +641,9 @@ const assetCacheRead = async function(assetKey, updateReadTime = false) {
}

return reportBack(bin[internalKey]);
};
}

const assetCacheWrite = async function(assetKey, details) {
async function assetCacheWrite(assetKey, details) {
let content = '';
let options = {};
if ( typeof details === 'string' ) {
Expand All @@ -603,6 +664,7 @@ const assetCacheWrite = async function(assetKey, details) {
entry = cacheDict[assetKey] = {};
}
entry.writeTime = entry.readTime = Date.now();
entry.resourceTime = options.resourceTime || 0;
if ( typeof options.url === 'string' ) {
entry.remoteURL = options.url;
}
Expand All @@ -617,9 +679,9 @@ const assetCacheWrite = async function(assetKey, details) {
fireNotification('after-asset-updated', result);
}
return result;
};
}

const assetCacheRemove = async function(pattern) {
async function assetCacheRemove(pattern) {
const cacheDict = await getAssetCacheRegistry();
const removedEntries = [];
const removedContent = [];
Expand All @@ -645,9 +707,39 @@ const assetCacheRemove = async function(pattern) {
assetKey: removedEntries[i]
});
}
};
}

async function assetCacheGetDetails(assetKey) {
const cacheDict = await getAssetCacheRegistry();
const entry = cacheDict[assetKey];
if ( entry === undefined ) { return; }
return entry;
}

const assetCacheMarkAsDirty = async function(pattern, exclude) {
async function assetCacheSetDetails(assetKey, details) {
const cacheDict = await getAssetCacheRegistry();
const entry = cacheDict[assetKey];
if ( entry === undefined ) { return; }
let modified = false;
for ( const [ k, v ] of Object.entries(details) ) {
if ( v === undefined ) {
if ( entry[k] !== undefined ) {
delete entry[k];
modified = true;
continue;
}
}
if ( v !== entry[k] ) {
entry[k] = v;
modified = true;
}
}
if ( modified ) {
saveAssetCacheRegistry();
}
}

async function assetCacheMarkAsDirty(pattern, exclude) {
const cacheDict = await getAssetCacheRegistry();
let mustSave = false;
for ( const assetKey in cacheDict ) {
Expand All @@ -673,13 +765,7 @@ const assetCacheMarkAsDirty = async function(pattern, exclude) {
if ( mustSave ) {
cacheStorage.set({ assetCacheRegistry });
}
};

/******************************************************************************/

const stringIsNotEmpty = function(s) {
return typeof s === 'string' && s !== '';
};
}

/*******************************************************************************
Expand Down Expand Up @@ -805,9 +891,17 @@ assets.get = async function(assetKey, options = {}) {

/******************************************************************************/

const getRemote = async function(assetKey) {
const assetRegistry = await getAssetSourceRegistry();
const assetDetails = assetRegistry[assetKey] || {};
async function getRemote(assetKey) {
const [
assetDetails = {},
cacheDetails = {},
] = await Promise.all([
assetSourceGetDetails(assetKey),
assetCacheGetDetails(assetKey),
]);

let error;
let stale = false;

const reportBack = function(content, err) {
const details = { assetKey, content };
Expand Down Expand Up @@ -865,27 +959,40 @@ const getRemote = async function(assetKey) {

// Failure
if ( stringIsNotEmpty(result.content) === false ) {
let error = result.statusText;
error = result.statusText;
if ( result.statusCode === 0 ) {
error = 'network error';
}
registerAssetSource(assetKey, {
error: { time: Date.now(), error }
});
continue;
}

error = undefined;

// If fetched resource is same older than cached one, ignore
stale = resourceIsStale(result, cacheDetails);
if ( stale ) { continue; }

// Success
assetCacheWrite(assetKey, {
content: result.content,
url: contentURL
url: contentURL,
resourceTime: result.resourceTime || 0,
});
registerAssetSource(assetKey, { error: undefined });
registerAssetSource(assetKey, { birthtime: undefined, error: undefined });
return reportBack(result.content);
}

return reportBack('', 'ENOTFOUND');
};
if ( error !== undefined ) {
registerAssetSource(assetKey, { error: { time: Date.now(), error } });
return reportBack('', 'ENOTFOUND');
}

if ( stale ) {
assetCacheSetDetails(assetKey, { writeTime: cacheDetails.resourceTime });
}

return reportBack('');
}

/******************************************************************************/

Expand Down Expand Up @@ -1052,13 +1159,13 @@ const updateNext = async function() {

remoteServerFriendly = false;

if ( result.content !== '' ) {
if ( result.error ) {
fireNotification('asset-update-failed', { assetKey: result.assetKey });
} else {
updaterUpdated.push(result.assetKey);
if ( result.assetKey === 'assets.json' ) {
if ( result.assetKey === 'assets.json' && result.content !== '' ) {
updateAssetSourceRegistry(result.content);
}
} else {
fireNotification('asset-update-failed', { assetKey: result.assetKey });
}

updaterTimer.on(updaterAssetDelay);
Expand All @@ -1072,7 +1179,7 @@ const updateDone = function() {
updaterUpdated.length = 0;
updaterStatus = undefined;
updaterAssetDelay = updaterAssetDelayDefault;
fireNotification('after-assets-updated', { assetKeys: assetKeys });
fireNotification('after-assets-updated', { assetKeys });
};

assets.updateStart = function(details) {
Expand Down

0 comments on commit 7daf313

Please sign in to comment.