✨ Progress on #15

.vscode/launch.json

@@ -10,15 +10,16 @@
             "name": "Launch Program",
             "program": "${workspaceFolder}/bin/mwoffliner.script.js",
             "args": [
-                "--mwUrl=https://bm.wikipedia.org",
+                "--mwUrl=https://en.wikipedia.org",
                 "--adminEmail=admin@kiwix.com",
                 "--speed=10",
                 // "--verbose",
                 // "--format=nozim",
-                // "--articleList=./articleList",
+                "--articleList=./articleList",
                 // "--articleList=https://ftp.nluug.nl/pub/kiwix/wp1/enwiki_2019-01/tops/100",
                 // "--articleList=https://download.kiwix.org/wp1/enwiki_2019-04/projects/Football",
-                "--redis=redis://127.0.0.1:6379"
+                "--redis=redis://127.0.0.1:6379",
+                // "--getCategories=true"
                 // "--customZimFavicon=test.png"
             ]
         }

res/style.css

@@ -72,4 +72,49 @@ h1[tabindex="0"].collapsible-heading>.indicator {
 
 .open-block {
     display: block;
+}
+
+.mwo-catlinks {
+    border: 1px solid #a2a9b1;
+    background-color: #f8f9fa;
+    padding: 5px;
+    margin-top: 1em;
+    clear: both;
+}
+
+.mwo-catlinks ul {
+    display: inline;
+    margin: 0;
+    padding: 0;
+    list-style: none none;
+}
+
+.mwo-catlinks li {
+    display: inline-block;
+    line-height: 1.25em;
+    border-left: 1px solid #a2a9b1;
+    margin: 0.125em 0;
+    padding: 0 0.5em;
+    zoom: 1;
+}
+
+.mwo-catlinks li:first-child {
+    padding-left: 0.25em;
+    border-left: 0;
+}
+
+.mwo-groups {
+    -webkit-column-count: 3;
+    -moz-column-count: 3;
+    column-count: 3;
+    -webkit-column-width: 24em;
+    -moz-column-width: 24em;
+    column-width: 24em;
+}
+
+.mwo-groups h3 {
+    line-height: 1.6;
+    margin-top: 0.3em;
+    margin-bottom: 0;
+    padding-bottom: 0;
 }

res/templates/categories.html

@@ -1,11 +1,11 @@
-<h2 class="section-heading in-block collapsible-heading" tabindex="0" aria-haspopup="true" data-section-id="Categories">
-    <div class="mw-ui-icon mw-ui-icon-arrow mw-ui-icon-element indicator" title=""></div>
-    <span class="mw-headline">Categories:</span>
-</h2>
-<div class="mf-section-{{ section_id }}" aria-pressed="true" aria-expanded="true">
+<br />
+<div class="mwo-catlinks">
+    Categories:
     <ul>
         {% for category in categories %}
-        <li><a href="{{category.url}}" rel="mw:WikiLink">{{category.name}}</a></li>
+        <li>
+            <a href="{{category.url}}" rel="mw:WikiLink">{{category.name}}</a>
+        </li>
         {% endfor %}
     </ul>
 </div>

res/templates/subcategories.html

@@ -1,11 +1,21 @@
-<h2 class="section-heading in-block collapsible-heading" tabindex="0" aria-haspopup="true" data-section-id="Categories">
+<h2 class="section-heading in-block collapsible-heading" tabindex="0" aria-haspopup="true"
+    data-section-id="SubCategories">
     <div class="mw-ui-icon mw-ui-icon-arrow mw-ui-icon-element indicator" title=""></div>
     <span class="mw-headline">Sub-Categories:</span>
 </h2>
 <div class="mf-section-{{ section_id }}" aria-pressed="true" aria-expanded="true">
-    <ul>
-        {% for category in subCategories %}
-        <li><a href="{{category.url}}" rel="mw:WikiLink">{{category.name}}</a></li>
-        {% endfor %}
-    </ul>
+    <div class="mwo-groups">
+        <ul>
+            {% for categoryGroup in groups %}
+            <div>
+                <h3>{{categoryGroup.title}}</h3>
+                <ul>
+                    {% for category in categoryGroup.items %}
+                    <li><a href="{{category.url}}" rel="mw:WikiLink">{{category.name}}</a></li>
+                    {% endfor %}
+                </ul>
+            </div>
+            {% endfor %}
+        </ul>
+    </div>
 </div>

res/templates/subpages.html

@@ -0,0 +1,20 @@
+<h2 class="section-heading in-block collapsible-heading" tabindex="0" aria-haspopup="true" data-section-id="Page">
+    <div class="mw-ui-icon mw-ui-icon-arrow mw-ui-icon-element indicator" title=""></div>
+    <span class="mw-headline">Pages:</span>
+</h2>
+<div class="mf-section-{{ section_id }}" aria-pressed="true" aria-expanded="true">
+    <div class="mwo-groups">
+        <ul>
+            {% for pageGroup in groups %}
+            <div>
+                <h3>{{pageGroup.title}}</h3>
+                <ul>
+                    {% for page in pageGroup.items %}
+                    <li><a href="{{page.url}}" rel="mw:WikiLink">{{page.name}}</a></li>
+                    {% endfor %}
+                </ul>
+            </div>
+            {% endfor %}
+        </ul>
+    </div>
+</div>

src/Downloader.ts

@@ -223,7 +223,7 @@ class Downloader {
       ? `${this.parsoidFallbackUrl}${encodeURIComponent(articleId)}`
       : `${this.mcsUrl}${encodeURIComponent(articleId)}`;
 
-    logger.log(`Getting ${useParsoidFallback ? 'desktop' : 'mobile'} article from ${articleApiUrl}`);
+    logger.info(`Getting ${useParsoidFallback ? 'desktop' : 'mobile'} article from ${articleApiUrl}`);
 
     try {
       const articleDetail = await articleDetailXId.get(articleId);

src/Templates.ts

@@ -15,6 +15,7 @@ const sectionTemplate = swig.compile(readTemplate(config.output.templates.sectio
 const subSectionTemplate = swig.compile(readTemplate(config.output.templates.subsection_wrapper));
 const categoriesTemplate = swig.compile(readTemplate(config.output.templates.categories));
 const subCategoriesTemplate = swig.compile(readTemplate(config.output.templates.subCategories));
+const subPagesTemplate = swig.compile(readTemplate(config.output.templates.subPages));
 
 const htmlTemplateCode = (articleId: string) => {
     const cssLinks = config.output.cssResources.reduce((buf, css) => {
@@ -41,4 +42,5 @@ export {
     articleListHomeTemplate,
     categoriesTemplate,
     subCategoriesTemplate,
+    subPagesTemplate,
 };

src/config.ts

@@ -84,6 +84,8 @@ const config = {
 
       subCategories: './templates/subcategories.html',
 
+      subPages: './templates/subpages.html',
+
       articleListHomeTemplate: './templates/article_list_home.html',
 
       /* License footer template code */

src/types.d.ts

@@ -70,6 +70,7 @@ interface PageInfo {
 type ArticleDetail = PageInfo & {
     subCategories?: PageInfo[],
     categories?: PageInfo[],
+    pages?: PageInfo[],
     thumbnail?: {
         source: string,
         height: number,

src/util/articleRenderers.ts

@@ -1,4 +1,4 @@
-import { leadSectionTemplate, sectionTemplate, subSectionTemplate, categoriesTemplate, subCategoriesTemplate } from '../Templates';
+import { leadSectionTemplate, sectionTemplate, subSectionTemplate, categoriesTemplate, subCategoriesTemplate, subPagesTemplate } from '../Templates';
 import { Dump } from '../Dump';
 import logger from '../Logger';
 
@@ -68,30 +68,72 @@ export function renderMCSArticle(json: any, dump: Dump, articleId: string, artic
     const resourceNamespace = 'A';
     const slashesInUrl = articleId.split('/').length - 1;
     const upStr = '../'.repeat(slashesInUrl + 1);
-    if (articleDetail.categories && articleDetail.categories.length) {
-        const categories = articleDetail.categories.map((category) => {
+    if (articleDetail.subCategories && articleDetail.subCategories.length) {
+        const subCategoriesAlphabetical = articleDetail.subCategories.map((category) => {
             return {
-                name: category.title,
+                name: category.title.split(':').slice(1).join(':'),
                 url: `${upStr}${resourceNamespace}/${category.title.replace(/ /g, '_')}${dump.nozim ? '.html' : ''}`,
             };
         });
-        html += categoriesTemplate({
+
+        const subCategories = articleDetail.subCategories.map((category) => {
+            return {
+                name: category.title.split(':').slice(1).join(':'),
+                url: `${upStr}${resourceNamespace}/${category.title.replace(/ /g, '_')}${dump.nozim ? '.html' : ''}`,
+            };
+        });
+
+        const groups = groupAlphabetical(subCategories);
+
+        html += subCategoriesTemplate({
             strings: dump.strings,
-            categories,
+            groups,
         });
     }
-    if (articleDetail.subCategories && articleDetail.subCategories.length) {
-        const subCategories = articleDetail.subCategories.map((category) => {
+    if (articleDetail.pages && articleDetail.pages.length) {
+        const pages = articleDetail.pages.map((page) => {
             return {
-                name: category.title,
+                name: page.title,
+                url: `${upStr}${resourceNamespace}/${page.title.replace(/ /g, '_')}${dump.nozim ? '.html' : ''}`,
+            };
+        });
+
+        const groups = groupAlphabetical(pages);
+
+        html += subPagesTemplate({
+            strings: dump.strings,
+            groups,
+        });
+    }
+    if (articleDetail.categories && articleDetail.categories.length) {
+        const categories = articleDetail.categories.map((category) => {
+            return {
+                name: category.title.split(':').slice(1).join(':'),
                 url: `${upStr}${resourceNamespace}/${category.title.replace(/ /g, '_')}${dump.nozim ? '.html' : ''}`,
             };
         });
-        html += subCategoriesTemplate({
+        html += categoriesTemplate({
             strings: dump.strings,
-            subCategories,
+            categories,
         });
     }
     html = html.replace(`__SUB_LEVEL_SECTION_${json.remaining.sections.length}__`, ''); // remove the last subcestion anchor (all other anchor are removed in the forEach)
     return html;
 }
+
+function groupAlphabetical(items: Array<{ name: string, url: string }>) {
+    const groupsAlphabetical = items.reduce((acc: any, item) => {
+        const groupId = item.name[0].toLocaleUpperCase();
+        acc[groupId] = (acc[groupId] || []).concat(item);
+        return acc;
+    }, {});
+
+    return Object.keys(groupsAlphabetical)
+        .sort()
+        .map((letter) => {
+            return {
+                title: letter,
+                items: groupsAlphabetical[letter],
+            };
+        });
+}

src/util/misc.ts

@@ -292,3 +292,11 @@ export function getStrippedTitleFromHtml(html: string) {
     return '';
   }
 }
+
+export function zip(...args: any[][]) {
+  const len = Math.max(...args.map((arr) => arr.length));
+  return ','.repeat(len).split(',')
+    .map((_, i) => {
+      return args.map((arr) => arr[i]);
+    });
+}