Merge pull request #1826 from openzim/format-parameter-fixes

Clearing Redis media storage after each dump

src/mwoffliner.lib.ts

@@ -346,6 +346,7 @@ async function execute(argv: any) {
     } else {
       try {
         await doDump(dump)
+        await filesToDownloadXPath.flush()
       } catch (err) {
         debugger
         throw err

src/util/dump.ts

@@ -45,7 +45,7 @@ export async function getAndProcessStylesheets(downloader: Downloader, redisStor
         while ((match = cssUrlRegexp.exec(body))) {
           let url = match[1]
 
-          /* Avoid 'data', so no url dependency */
+          /* Avoid 'data', so no URL dependency */
           if (!url.match('^data')) {
             const filePathname = urlParser.parse(url, false, true).pathname
             if (filePathname) {

test/e2e/multimediaContent.test.ts

@@ -0,0 +1,124 @@
+import * as mwoffliner from '../../src/mwoffliner.lib.js'
+import { execa } from 'execa'
+import rimraf from 'rimraf'
+import { zimcheckAvailable, zimcheck, zimdumpAvailable, zimdump } from '../util.js'
+import 'dotenv/config'
+import { jest } from '@jest/globals'
+
+jest.setTimeout(60000)
+
+describe('Multimedia', () => {
+  const now = new Date()
+  const testId = `mwo-test-${+now}`
+
+  const parameters = {
+    mwUrl: 'https://en.m.wikipedia.org',
+    adminEmail: 'test@kiwix.org',
+    articleList: 'User:Kelson/MWoffliner_CI_reference',
+    outputDirectory: testId,
+    redis: process.env.REDIS,
+    customZimDescription: 'Example of the description',
+  }
+
+  test('check multimedia content from wikipedia test page', async () => {
+    await execa('redis-cli flushall', { shell: true })
+
+    const [dump] = await mwoffliner.execute(parameters)
+
+    expect(dump.status.articles.success).toEqual(1)
+    expect(dump.status.articles.fail).toEqual(0)
+
+    if (await zimcheckAvailable()) {
+      await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    if (await zimdumpAvailable()) {
+      const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
+
+      expect(mediaFiles.split('\n').sort()).toEqual(
+        [
+          'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+          'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
+          'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+          'I/Kiwix_icon.svg.png',
+          'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+          'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+          'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+        ].sort(),
+      )
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    rimraf.sync(`./${testId}`)
+    const redisScan = await execa('redis-cli --scan', { shell: true })
+    // Redis has been cleared
+    expect(redisScan.stdout).toEqual('')
+  })
+
+  test('check multimedia content from wikipedia test page with different formates', async () => {
+    await execa('redis-cli flushall', { shell: true })
+    const dumps = await mwoffliner.execute({ ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet'] })
+
+    expect(dumps).toHaveLength(4)
+    for (const dump of dumps) {
+      expect(dump.status.articles.success).toEqual(1)
+      expect(dump.status.articles.fail).toEqual(0)
+
+      if (await zimcheckAvailable()) {
+        await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
+      } else {
+        console.log('Zimcheck not installed, skipping test')
+      }
+
+      if (await zimdumpAvailable()) {
+        const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
+        if (dump.nopic) {
+          expect(mediaFiles.split('\n').sort()).toEqual(
+            [
+              'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+              // 'I/Kiwix_icon.svg.png',
+              // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+              // 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+              // 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+            ].sort(),
+          )
+        } else if (dump.novid) {
+          expect(mediaFiles.split('\n').sort()).toEqual(
+            [
+              'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+              'I/Kiwix_icon.svg.png',
+              // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+              'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+              'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+            ].sort(),
+          )
+        } else if (dump.nopdf) {
+          expect(mediaFiles.split('\n').sort()).toEqual(
+            [
+              // 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',  // this file was omitted by nopdf parameter
+              'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
+              'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+              'I/Kiwix_icon.svg.png',
+              'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+              'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+              'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+            ].sort(),
+          )
+        }
+      } else {
+        console.log('Zimcheck not installed, skipping test')
+      }
+    }
+    rimraf.sync(`./${testId}`)
+    const redisScan = await execa('redis-cli --scan', { shell: true })
+    // Redis has been cleared
+    expect(redisScan.stdout).toEqual('')
+  })
+})