Avoid generating a full build with each language

This patch changes the build so that bundling all languages takes 25 seconds instead of a million years. The approach is in-place substituting the locale strings using a template instead of regenerating the build for each language. It also adds a "languages" property to package.json that specifies which languages to build when running npm run build. Bug: T246778 Change-Id: I16ecd12b97326278b44c7e4e3afe0dc00a784991
wikimedia · Mar 12, 2020 · 9df72d8 · 9df72d8
1 parent 576e5c8
commit 9df72d8
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 50 deletions.
diff --git a/package.json b/package.json
@@ -73,6 +73,15 @@
         "Wikipedia",
         "Stats"
     ],
+    "languages": [
+        "en",
+        "zh-hant",
+        "lb",
+        "mk",
+        "pt-br",
+        "tr",
+        "uk"
+    ],
     "author": "Dan Andreescu, Francisco Dans, Marcel Ruiz Forns, Nuria Ruiz",
     "license": "MIT",
     "homepage": "https://github.com/milimetric/wikistats-prototype#readme"

diff --git a/src/components/detail/GraphPanel.vue b/src/components/detail/GraphPanel.vue
@@ -156,9 +156,6 @@ export default {
             aggregate: function () {
                 return this.graphModel && this.graphModel.getAggregate();
             },
-            changeOverRange: function () {
-                return this.graphModel.getChangeOverRange();
-            },
             activeBreakdown: function () {
                 return this.graphModel.activeBreakdown;
             },

diff --git a/src/models/GraphModel.js b/src/models/GraphModel.js
@@ -309,12 +309,6 @@ class GraphModel {
         return _.takeRight(values, limit);
     }
 
-    getChangeOverRange () {
-        const data = this.getAggregatedValues();
-        if (data[0] == 0 || data.length == 0) return null;
-        return ((data[data.length - 1] - data[0]) / data[0] * 100).toFixed(2);
-    }
-
     getActiveBreakdownValues () {
         const actives = this.activeBreakdown.values.filter(bv => bv.on).map(bv => bv.key);
         return actives.reduce((r, a) => { r[a] = true; return r; }, {});

diff --git a/webpack/build.js b/webpack/build.js
@@ -5,9 +5,16 @@ const path = require('path');
 const fs = require('fs');
 const _ = require('lodash');
 const webpack = require('webpack');
-const distPath = `${__dirname}/../dist`;
 const HtmlWebpackPlugin = require('html-webpack-plugin');
 
+const distPath = `${__dirname}/../dist`;
+const i18nSrcPath = `${__dirname}/../src/i18n`;
+
+const timei18nPath = path.join(__dirname, '../node_modules/date-fns/locale');
+const availableTimeLocales = new Set(fs.readdirSync(timei18nPath));
+const version = require("../package.json").version;
+const configLanguages = require("../package.json").languages;
+
 const deleteFolderRecursive = function(pathToDir) {
   if (fs.existsSync(pathToDir)) {
     fs.readdirSync(pathToDir).forEach((file, index) => {
@@ -23,61 +30,80 @@ const deleteFolderRecursive = function(pathToDir) {
 };
 
 const getLocalesToBuild = () => {
-    const i18ndir = path.join(__dirname, '../src/i18n/');
-    let availableLocales = fs.readdirSync(i18ndir).map(file => file.replace('.json', ''));
+    let availableLocales = fs.readdirSync(i18nSrcPath).map(file => file.replace('.json', ''));
     const languagesIndex = process.argv.indexOf('languages');
     if (languagesIndex > -1) {
         const languages = process.argv[languagesIndex + 1].split(',');
         availableLocales = availableLocales.filter(loc => languages.includes(loc));
+    } else {
+        availableLocales = availableLocales.filter(loc => configLanguages.includes(loc));
     }
     return availableLocales
 }
 
+/**
+    Instead of passing a list of configurations, each with a different
+    language, we generate an initial build, get the contents of the
+    initial build file, and manually inject the locales with a template.
+**/
+
+const applyLanguageToBundle = (locale, bundle) => {
+    const timeLocaleName = availableTimeLocales.has(locale) ? locale : 'en-US';
+    const userMessages = JSON.stringify(require(`${i18nSrcPath}/${locale}.json`));
+    const timeLocale = JSON.stringify(require(`../node_modules/date-fns/locale/${timeLocaleName}`))
+    const fullFile = `userMessages=${userMessages};timeLocale=${timeLocale};${bundle}`;
+    fs.writeFileSync(`${distPath}/main.bundle.${version}.${locale}.js`, fullFile, function (err) {
+        if (err) {
+            console.log(`An error occured while writing ${locale} locale to file.`);
+            console.error(err);
+        }
+    });
+}
+
+const moveAssetsToV2Dir = () => {
+    const v2Path = `${distPath}/assets-v2`;
+    if (!fs.existsSync(v2Path)) {
+        fs.mkdirSync(v2Path);
+    }
+    fs.readdirSync(distPath)
+        .filter(item => item !== 'index.html' && item !== 'assets-v2')
+        .forEach(item => {
+            fs.renameSync(`${distPath}/${item}`, `${v2Path}/${item}`)
+        })
+    ;
+}
+
 deleteFolderRecursive(distPath);
 
-const configs = [];
 const locales = getLocalesToBuild();
-console.info(`Building locales ${locales.join(', ')}`);
-locales.forEach(locale => {
-    const config = _.cloneDeep(prodConfig);
-    const timei18ndir = path.join(__dirname, '../node_modules/date-fns/locale');
-    const availableTimeLocales = new Set(fs.readdirSync(timei18ndir));
-    const timeLocale = availableTimeLocales.has(locale) ? locale : 'en-US';
-    config.plugins.push(
-        new webpack.ProvidePlugin({
-            timeLocale: path.join(__dirname, `../node_modules/date-fns/locale/${timeLocale}`),
-            userMessages: path.join(__dirname, `../src/i18n/${locale}.json`)
-        }),
-        new webpack.DefinePlugin({
-            AVAILABLE_LANGUAGES: JSON.stringify(locales)
-        }),
-        new HtmlWebpackPlugin({
-            filename: 'index.html',
-            favicon: 'src/assets/analytics.png',
-            template: 'src/index.ejs',
-            inject: false,
-            availableLocales: locales
-        })
-    );
-    const version = require("../package.json").version;
-    config.output.filename = `[name].bundle.${version}.${locale}.js`;
-    configs.push(config);
-});
+console.info(`Building languages ${locales.join(', ')}`);
+const config = _.cloneDeep(prodConfig);
 
-webpack(configs, (err, stats) => {
+// Probably move all of this to production config
+config.plugins.push(
+    new webpack.DefinePlugin({
+        AVAILABLE_LANGUAGES: JSON.stringify(locales)
+    }),
+    new HtmlWebpackPlugin({
+        filename: 'index.html',
+        favicon: 'src/assets/analytics.png',
+        template: 'src/index.ejs',
+        inject: false,
+        availableLocales: locales
+    })
+);
+config.output.filename = `main.bundle.${version}.en.js`;
+
+webpack(config, (err, stats) => {
     console.log(stats);
     if (err || stats.hasErrors()) {
         console.error(stats.toJson().errors);
     } else {
-        const v2Path = `${distPath}/assets-v2`;
-        if (!fs.existsSync(v2Path)) {
-            fs.mkdirSync(v2Path);
-        }
-        fs.readdirSync(distPath)
-            .filter(item => item !== 'index.html' && item !== 'assets-v2')
-            .forEach(item => {
-                fs.renameSync(`${distPath}/${item}`, `${v2Path}/${item}`)
-            })
-        ;
+        const initialBundle = fs.readFileSync(`${distPath}/${config.output.filename}`);
+        locales.forEach(locale => {
+            console.log(`Generating ${locale} build`);
+            applyLanguageToBundle(locale, initialBundle);
+        });
+        moveAssetsToV2Dir();
     }
 });