Clearing the Redis storage with media files and CSS after each iterat…

…ion.
openzim · Apr 12, 2023 · b2e901b · b2e901b
1 parent ecca00a
commit b2e901b
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 34 deletions.
diff --git a/src/RedisStore.ts b/src/RedisStore.ts
@@ -8,7 +8,6 @@ class RedisStore implements RS {
  private storesReady: boolean
 
  private _filesToDownloadXPath: RKVS<FileDetail>
- private _mediaToDownloadXPath: RKVS<FileDetail>
  private _filesToRetryXPath: RKVS<FileDetail>
  private _articleDetailXId: RKVS<ArticleDetail>
  private _redirectsXId: RKVS<ArticleRedirect>
@@ -59,15 +58,8 @@ class RedisStore implements RS {
  }
  }
 
- public async flushMediaToDownloadXPath() {
- if (this._client.isReady && this.storesReady) {
- logger.log('Flushing Redis DB for storing media')
- await this._mediaToDownloadXPath.flush()
- }
- }
-
  public async checkForExistingStores() {
- const patterns = ['*-media', '*-files', '*-media-retry', '*-detail', '*-redirect']
+ const patterns = ['*-media', '*-media-retry', '*-detail', '*-redirect']
  let keys: string[] = []
  for (const pattern of patterns) {
  keys = keys.concat(await this._client.keys(pattern))
@@ -85,13 +77,7 @@ class RedisStore implements RS {
  }
 
  private async populateStores() {
- this._mediaToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-media`, {
- u: 'url',
- n: 'namespace',
- m: 'mult',
- w: 'width',
- })
- this._filesToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-files`, {
+ this._filesToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-media`, {
  u: 'url',
  n: 'namespace',
  m: 'mult',
@@ -133,10 +119,6 @@ class RedisStore implements RS {
  return this._filesToDownloadXPath
  }
 
- public get mediaToDownloadXPath(): RKVS<FileDetail> {
- return this._mediaToDownloadXPath
- }
-
  public get filesToRetryXPath(): RKVS<FileDetail> {
  return this._filesToRetryXPath
  }

diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts
@@ -207,7 +207,7 @@ async function execute(argv: any) {
 
  const redisStore = new RedisStore(argv.redis || config.defaults.redisPath)
  await redisStore.connect()
- const { articleDetailXId, filesToDownloadXPath, mediaToDownloadXPath, filesToRetryXPath, redirectsXId } = redisStore
+ const { articleDetailXId, filesToDownloadXPath, filesToRetryXPath, redirectsXId } = redisStore
 
  // Output directory
  const outputDirectory = path.isAbsolute(_outputDirectory || '') ? _outputDirectory : path.join(process.cwd(), _outputDirectory || 'out')
@@ -344,7 +344,7 @@ async function execute(argv: any) {
  } else {
  try {
  await doDump(dump)
- await mediaToDownloadXPath.flush()
+ await filesToDownloadXPath.flush()
  } catch (err) {
  debugger
  throw err
@@ -439,7 +439,6 @@ async function execute(argv: any) {
  )
 
  await downloadFiles(filesToDownloadXPath, filesToRetryXPath, zimCreator, dump, downloader)
- await downloadFiles(mediaToDownloadXPath, filesToRetryXPath, zimCreator, dump, downloader)
 
  logger.log('Writing Article Redirects')
  await writeArticleRedirects(downloader, dump, zimCreator)
@@ -616,7 +615,7 @@ async function execute(argv: any) {
  articleDetail.internalThumbnailUrl = getRelativeFilePath('Main_Page', getMediaBase(suitableResUrl, true), 'I')
 
  await Promise.all([
- mediaToDownloadXPath.set(path, { url: downloader.serializeUrl(suitableResUrl), mult, width } as FileDetail),
+ filesToDownloadXPath.set(path, { url: downloader.serializeUrl(suitableResUrl), mult, width } as FileDetail),
  articleDetailXId.set(articleId, articleDetail),
  ])
  articlesWithImages++

diff --git a/src/types.d.ts b/src/types.d.ts
@@ -69,7 +69,6 @@ interface RKVS<T> {
 // RedisStore Interface
 interface RS {
  readonly client: any // RedisClientType
- readonly mediaToDownloadXPath: RKVS<FileDetail>
  readonly filesToDownloadXPath: RKVS<FileDetail>
  readonly filesToRetryXPath: RKVS<FileDetail>
  readonly articleDetailXId: RKVS<ArticleDetail>

diff --git a/src/util/dump.ts b/src/util/dump.ts
@@ -58,8 +58,7 @@ export async function getAndProcessStylesheets(downloader: Downloader, redisStor
  url = getFullUrl(url, cssUrl)
  url = url.indexOf('%') < 0 ? encodeURI(url) : url
 
- /* Download CSS dependency, but avoid duplicate calls */
- if (!downloader.cssDependenceUrls.hasOwnProperty(url) && filename) {
+ if (filename) {
  downloader.cssDependenceUrls[url] = true
  filesToDownloadXPath.set(config.output.dirs.mediawiki + '/' + filename, { url: downloader.serializeUrl(url), namespace: '-' })
  }

diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts
@@ -193,21 +193,21 @@ async function saveArticle(
  try {
  const { finalHTML, mediaDependencies, subtitles } = await processArticleHtml(articleHtml, redisStore, mw, dump, articleId, articleDetail, _moduleDependencies, downloader.webp)
 
- const mediaToDownload: KVS<FileDetail> = {}
+ const filesToDownload: KVS<FileDetail> = {}
 
  subtitles.forEach((s) => {
- mediaToDownload[s.path] = { url: s.url, namespace: '-' }
+ filesToDownload[s.path] = { url: s.url, namespace: '-' }
  })
 
  if (mediaDependencies.length) {
- const existingVals = await redisStore.mediaToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path))
+ const existingVals = await redisStore.filesToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path))
 
  for (const dep of mediaDependencies) {
  const { mult, width } = getSizeFromUrl(dep.url)
  const existingVal = existingVals[dep.path]
  const currentDepIsHigherRes = !existingVal || existingVal.width < (width || 10e6) || existingVal.mult < (mult || 1)
  if (currentDepIsHigherRes) {
- mediaToDownload[dep.path] = {
+ filesToDownload[dep.path] = {
  url: downloader.serializeUrl(dep.url),
  mult,
  width,
@@ -216,7 +216,7 @@ async function saveArticle(
  }
  }
 
- await redisStore.mediaToDownloadXPath.setMany(mediaToDownload)
+ await redisStore.filesToDownloadXPath.setMany(filesToDownload)
 
  const zimArticle = new ZimArticle({
  url: articleId,

diff --git a/test/unit/bootstrap.ts b/test/unit/bootstrap.ts
@@ -9,8 +9,8 @@ export const redisStore = new RedisStore(process.env.REDIS || config.defaults.re
 
 export const startRedis = async () => {
  await redisStore.connect()
- const { articleDetailXId, redirectsXId, filesToDownloadXPath, mediaToDownloadXPath, filesToRetryXPath } = redisStore
- await Promise.all([articleDetailXId.flush(), redirectsXId.flush(), mediaToDownloadXPath.flush(), filesToDownloadXPath.flush(), filesToRetryXPath.flush()])
+ const { articleDetailXId, redirectsXId, filesToDownloadXPath, filesToRetryXPath } = redisStore
+ await Promise.all([articleDetailXId.flush(), redirectsXId.flush(), filesToDownloadXPath.flush(), filesToRetryXPath.flush()])
 }
 
 export const stopRedis = async () => {