diff --git a/src/RedisStore.ts b/src/RedisStore.ts index bef017b97..8206e27ae 100644 --- a/src/RedisStore.ts +++ b/src/RedisStore.ts @@ -8,7 +8,6 @@ class RedisStore implements RS { private storesReady: boolean private _filesToDownloadXPath: RKVS - private _mediaToDownloadXPath: RKVS private _filesToRetryXPath: RKVS private _articleDetailXId: RKVS private _redirectsXId: RKVS @@ -59,15 +58,8 @@ class RedisStore implements RS { } } - public async flushMediaToDownloadXPath() { - if (this._client.isReady && this.storesReady) { - logger.log('Flushing Redis DB for storing media') - await this._mediaToDownloadXPath.flush() - } - } - public async checkForExistingStores() { - const patterns = ['*-media', '*-files', '*-media-retry', '*-detail', '*-redirect'] + const patterns = ['*-media', '*-media-retry', '*-detail', '*-redirect'] let keys: string[] = [] for (const pattern of patterns) { keys = keys.concat(await this._client.keys(pattern)) @@ -85,13 +77,7 @@ class RedisStore implements RS { } private async populateStores() { - this._mediaToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-media`, { - u: 'url', - n: 'namespace', - m: 'mult', - w: 'width', - }) - this._filesToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-files`, { + this._filesToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-media`, { u: 'url', n: 'namespace', m: 'mult', @@ -133,10 +119,6 @@ class RedisStore implements RS { return this._filesToDownloadXPath } - public get mediaToDownloadXPath(): RKVS { - return this._mediaToDownloadXPath - } - public get filesToRetryXPath(): RKVS { return this._filesToRetryXPath } diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index 818cf4054..4c4c6445c 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -208,7 +208,7 @@ async function execute(argv: any) { const redisStore = new RedisStore(argv.redis || config.defaults.redisPath) await redisStore.connect() - const { articleDetailXId, filesToDownloadXPath, mediaToDownloadXPath, filesToRetryXPath, redirectsXId } = redisStore + const { articleDetailXId, filesToDownloadXPath, filesToRetryXPath, redirectsXId } = redisStore // Output directory const outputDirectory = path.isAbsolute(_outputDirectory || '') ? _outputDirectory : path.join(process.cwd(), _outputDirectory || 'out') @@ -346,7 +346,7 @@ async function execute(argv: any) { } else { try { await doDump(dump) - await mediaToDownloadXPath.flush() + await filesToDownloadXPath.flush() } catch (err) { debugger throw err @@ -441,7 +441,6 @@ async function execute(argv: any) { ) await downloadFiles(filesToDownloadXPath, filesToRetryXPath, zimCreator, dump, downloader) - await downloadFiles(mediaToDownloadXPath, filesToRetryXPath, zimCreator, dump, downloader) logger.log('Writing Article Redirects') await writeArticleRedirects(downloader, dump, zimCreator) @@ -618,7 +617,7 @@ async function execute(argv: any) { articleDetail.internalThumbnailUrl = getRelativeFilePath('Main_Page', getMediaBase(suitableResUrl, true), 'I') await Promise.all([ - mediaToDownloadXPath.set(path, { url: downloader.serializeUrl(suitableResUrl), mult, width } as FileDetail), + filesToDownloadXPath.set(path, { url: downloader.serializeUrl(suitableResUrl), mult, width } as FileDetail), articleDetailXId.set(articleId, articleDetail), ]) articlesWithImages++ diff --git a/src/types.d.ts b/src/types.d.ts index cb891729d..50ffb8156 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -69,7 +69,6 @@ interface RKVS { // RedisStore Interface interface RS { readonly client: any // RedisClientType - readonly mediaToDownloadXPath: RKVS readonly filesToDownloadXPath: RKVS readonly filesToRetryXPath: RKVS readonly articleDetailXId: RKVS diff --git a/src/util/dump.ts b/src/util/dump.ts index 24fef9605..a6180a06e 100644 --- a/src/util/dump.ts +++ b/src/util/dump.ts @@ -58,8 +58,7 @@ export async function getAndProcessStylesheets(downloader: Downloader, redisStor url = getFullUrl(url, cssUrl) url = url.indexOf('%') < 0 ? encodeURI(url) : url - /* Download CSS dependency, but avoid duplicate calls */ - if (!downloader.cssDependenceUrls.hasOwnProperty(url) && filename) { + if (filename) { downloader.cssDependenceUrls[url] = true filesToDownloadXPath.set(config.output.dirs.mediawiki + '/' + filename, { url: downloader.serializeUrl(url), namespace: '-' }) } diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts index 76f24a97f..93bfb5db9 100644 --- a/src/util/saveArticles.ts +++ b/src/util/saveArticles.ts @@ -193,21 +193,21 @@ async function saveArticle( try { const { finalHTML, mediaDependencies, subtitles } = await processArticleHtml(articleHtml, redisStore, mw, dump, articleId, articleDetail, _moduleDependencies, downloader.webp) - const mediaToDownload: KVS = {} + const filesToDownload: KVS = {} subtitles.forEach((s) => { - mediaToDownload[s.path] = { url: s.url, namespace: '-' } + filesToDownload[s.path] = { url: s.url, namespace: '-' } }) if (mediaDependencies.length) { - const existingVals = await redisStore.mediaToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path)) + const existingVals = await redisStore.filesToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path)) for (const dep of mediaDependencies) { const { mult, width } = getSizeFromUrl(dep.url) const existingVal = existingVals[dep.path] const currentDepIsHigherRes = !existingVal || existingVal.width < (width || 10e6) || existingVal.mult < (mult || 1) if (currentDepIsHigherRes) { - mediaToDownload[dep.path] = { + filesToDownload[dep.path] = { url: downloader.serializeUrl(dep.url), mult, width, @@ -216,7 +216,7 @@ async function saveArticle( } } - await redisStore.mediaToDownloadXPath.setMany(mediaToDownload) + await redisStore.filesToDownloadXPath.setMany(filesToDownload) const zimArticle = new ZimArticle({ url: articleId, diff --git a/test/unit/bootstrap.ts b/test/unit/bootstrap.ts index 10b0c18ec..9875b089c 100644 --- a/test/unit/bootstrap.ts +++ b/test/unit/bootstrap.ts @@ -9,8 +9,8 @@ export const redisStore = new RedisStore(process.env.REDIS || config.defaults.re export const startRedis = async () => { await redisStore.connect() - const { articleDetailXId, redirectsXId, filesToDownloadXPath, mediaToDownloadXPath, filesToRetryXPath } = redisStore - await Promise.all([articleDetailXId.flush(), redirectsXId.flush(), mediaToDownloadXPath.flush(), filesToDownloadXPath.flush(), filesToRetryXPath.flush()]) + const { articleDetailXId, redirectsXId, filesToDownloadXPath, filesToRetryXPath } = redisStore + await Promise.all([articleDetailXId.flush(), redirectsXId.flush(), filesToDownloadXPath.flush(), filesToRetryXPath.flush()]) } export const stopRedis = async () => {