Merge branch 'develop'

CHANGELOG.md

@@ -1,3 +1,26 @@
+# 1.8.13
+
+### Notable fixes
+
+* Fixed a bug in the safeRun.sh script (#4935)
+* Don't create sessions on some static resources (#4921)
+* Fixed issue with non-opening device keyboard on smartphones (#4929)
+* Add version string to iframe_editor.css to prevent stale cache entry (#4964)
+
+### Notable enhancements
+
+* Refactor pad loading (no document.write anymore) (#4960)
+* Improve import/export functionality, logging and tests (#4957)
+* Refactor CSS manager creation (#4963)
+* Better metrics
+* Add test for client height (#4965)
+
+### Dependencies
+
+* ueberDB2 1.3.2 -> 1.4.4
+* express-rate-limit 5.2.5 -> 5.2.6
+* etherpad-require-kernel 1.0.9 -> 1.0.11
+
 # 1.8.12
 
 Special mention: Thanks to Sauce Labs for additional testing tunnels to help us grow! :)

README.md

@@ -15,6 +15,7 @@ Etherpad is extremely flexible providing you the means to modify it to solve wha
 * [Video Chat](https://video.etherpad.com) - Plugins to enable Video and Audio chat in a pad.
 * [Collaboration++](https://collab.etherpad.com) - Plugins to improve the really-real time collaboration experience, suitable for busy pads.
 * [Document Analysis](https://analysis.etherpad.com) - Plugins to improve author and document analysis during and post creation.
+* [Scale](https://shard.etherpad.com) - Etherpad running at scale with pad sharding which allows Etherpad to scale to ∞ number of Active Pads with up to ~20,000 edits per second, per pad.
 
 # Project Status
 

src/bin/safeRun.sh

@@ -24,8 +24,8 @@ fatal() { error "$@"; exit 1; }
 LAST_EMAIL_SEND=0
 
 # Move to the Etherpad base directory.
-MY_DIR=$(try cd "${0%/*}" && try pwd -P) || exit 1
-try cd "${MY_DIR}/../.."
+MY_DIR=$(cd "${0%/*}" && pwd -P) || exit 1
+cd "${MY_DIR}/../.." || exit 1
 
 # Check if a logfile parameter is set
 LOG="$1"

src/locales/ku-latn.json

@@ -31,7 +31,7 @@
 	"pad.settings.stickychat": "Di ekranê de hertim çet bike",
 	"pad.settings.chatandusers": "Çeta û Bikarhênera Nîşan bide",
 	"pad.settings.colorcheck": "Rengên nivîskarîye",
-	"pad.settings.linenocheck": "Hejmarên rêze",
+	"pad.settings.linenocheck": "Hejmarên rêzê",
 	"pad.settings.rtlcheck": "Bila naverok ji raste ber bi çepe be xwendin?",
 	"pad.settings.fontType": "Tîpa nivîsê:",
 	"pad.settings.language": "Ziman:",

src/node/db/DB.js

@@ -24,6 +24,7 @@
 const ueberDB = require('ueberdb2');
 const settings = require('../utils/Settings');
 const log4js = require('log4js');
+const stats = require('../stats');
 const util = require('util');
 
 // set database settings
@@ -48,6 +49,13 @@ exports.init = async () => await new Promise((resolve, reject) => {
       process.exit(1);
     }
 
+    if (db.metrics != null) {
+      for (const [metric, value] of Object.entries(db.metrics)) {
+        if (typeof value !== 'number') continue;
+        stats.gauge(`ueberdb_${metric}`, () => db.metrics[metric]);
+      }
+    }
+
     // everything ok, set up Promise-based methods
     ['get', 'set', 'findKeys', 'getSub', 'setSub', 'remove'].forEach((fn) => {
       exports[fn] = util.promisify(db[fn].bind(db));

src/node/handler/ExportHandler.js

@@ -33,24 +33,12 @@ const util = require('util');
 const fsp_writeFile = util.promisify(fs.writeFile);
 const fsp_unlink = util.promisify(fs.unlink);
 
-let convertor = null;
-
-// load abiword only if it is enabled
-if (settings.abiword != null) {
-  convertor = require('../utils/Abiword');
-}
-
-// Use LibreOffice if an executable has been defined in the settings
-if (settings.soffice != null) {
-  convertor = require('../utils/LibreOffice');
-}
-
 const tempDirectory = os.tmpdir();
 
 /**
  * do a requested export
  */
-const doExport = async (req, res, padId, readOnlyId, type) => {
+exports.doExport = async (req, res, padId, readOnlyId, type) => {
   // avoid naming the read-only file as the original pad's id
   let fileName = readOnlyId ? readOnlyId : padId;
 
@@ -85,7 +73,7 @@ const doExport = async (req, res, padId, readOnlyId, type) => {
       const newHTML = await hooks.aCallFirst('exportHTMLSend', html);
       if (newHTML.length) html = newHTML;
       res.send(html);
-      throw 'stop';
+      return;
     }
 
     // else write the html export to a file
@@ -98,20 +86,19 @@ const doExport = async (req, res, padId, readOnlyId, type) => {
     html = null;
     await TidyHtml.tidy(srcFile);
 
-    // send the convert job to the convertor (abiword, libreoffice, ..)
+    // send the convert job to the converter (abiword, libreoffice, ..)
     const destFile = `${tempDirectory}/etherpad_export_${randNum}.${type}`;
 
     // Allow plugins to overwrite the convert in export process
     const result = await hooks.aCallAll('exportConvert', {srcFile, destFile, req, res});
     if (result.length > 0) {
       // console.log("export handled by plugin", destFile);
     } else {
-      // @TODO no Promise interface for convertors (yet)
-      await new Promise((resolve, reject) => {
-        convertor.convertFile(srcFile, destFile, type, (err) => {
-          err ? reject('convertFailed') : resolve();
-        });
-      });
+      const converter =
+          settings.soffice != null ? require('../utils/LibreOffice')
+          : settings.abiword != null ? require('../utils/Abiword')
+          : null;
+      await converter.convertFile(srcFile, destFile, type);
     }
 
     // send the file
@@ -128,11 +115,3 @@ const doExport = async (req, res, padId, readOnlyId, type) => {
     await fsp_unlink(destFile);
   }
 };
-
-exports.doExport = (req, res, padId, readOnlyId, type) => {
-  doExport(req, res, padId, readOnlyId, type).catch((err) => {
-    if (err !== 'stop') {
-      throw err;
-    }
-  });
-};

src/node/handler/ImportHandler.js

@@ -55,17 +55,17 @@ const rm = async (path) => {
   }
 };
 
-let convertor = null;
+let converter = null;
 let exportExtension = 'htm';
 
 // load abiword only if it is enabled and if soffice is disabled
 if (settings.abiword != null && settings.soffice == null) {
-  convertor = require('../utils/Abiword');
+  converter = require('../utils/Abiword');
 }
 
 // load soffice only if it is enabled
 if (settings.soffice != null) {
-  convertor = require('../utils/LibreOffice');
+  converter = require('../utils/LibreOffice');
   exportExtension = 'html';
 }
 
@@ -80,8 +80,8 @@ const doImport = async (req, res, padId) => {
   // set html in the pad
   const randNum = Math.floor(Math.random() * 0xFFFFFFFF);
 
-  // setting flag for whether to use convertor or not
-  let useConvertor = (convertor != null);
+  // setting flag for whether to use converter or not
+  let useConverter = (converter != null);
 
   const form = new formidable.IncomingForm();
   form.keepExtensions = true;
@@ -170,30 +170,25 @@ const doImport = async (req, res, padId) => {
   // convert file to html if necessary
   if (!importHandledByPlugin && !directDatabaseAccess) {
     if (fileIsTXT) {
-      // Don't use convertor for text files
-      useConvertor = false;
+      // Don't use converter for text files
+      useConverter = false;
     }
 
     // See https://github.com/ether/etherpad-lite/issues/2572
-    if (fileIsHTML || !useConvertor) {
-      // if no convertor only rename
+    if (fileIsHTML || !useConverter) {
+      // if no converter only rename
       await fs.rename(srcFile, destFile);
     } else {
-      // @TODO - no Promise interface for convertors (yet)
-      await new Promise((resolve, reject) => {
-        convertor.convertFile(srcFile, destFile, exportExtension, (err) => {
-          // catch convert errors
-          if (err) {
-            logger.warn(`Converting Error: ${err.stack || err}`);
-            return reject(new ImportError('convertFailed'));
-          }
-          resolve();
-        });
-      });
+      try {
+        await converter.convertFile(srcFile, destFile, exportExtension);
+      } catch (err) {
+        logger.warn(`Converting Error: ${err.stack || err}`);
+        throw new ImportError('convertFailed');
+      }
     }
   }
 
-  if (!useConvertor && !directDatabaseAccess) {
+  if (!useConverter && !directDatabaseAccess) {
     // Read the file with no encoding for raw buffer access.
     const buf = await fs.readFile(destFile);
 
@@ -224,7 +219,7 @@ const doImport = async (req, res, padId) => {
 
   // change text of the pad and broadcast the changeset
   if (!directDatabaseAccess) {
-    if (importHandledByPlugin || useConvertor || fileIsHTML) {
+    if (importHandledByPlugin || useConverter || fileIsHTML) {
       try {
         await importHtml.setPadHTML(pad, text);
       } catch (err) {

src/node/hooks/express/importexport.js

@@ -60,7 +60,7 @@ exports.expressCreateServer = (hookName, args, cb) => {
         }
 
         console.log(`Exporting pad "${req.params.pad}" in ${req.params.type} format`);
-        exportHandler.doExport(req, res, padId, readOnlyId, req.params.type);
+        await exportHandler.doExport(req, res, padId, readOnlyId, req.params.type);
       }
     })().catch((err) => next(err || new Error(err)));
   });

src/node/hooks/express/webaccess.js

@@ -10,12 +10,20 @@ const readOnlyManager = require('../../db/ReadOnlyManager');
 hooks.deprecationNotices.authFailure = 'use the authnFailure and authzFailure hooks instead';
 
 const staticPathsRE = new RegExp(`^/(?:${[
-  'api/.*',
+  'api(?:/.*)?',
   'favicon\\.ico',
+  'ep/pad/connection-diagnostic-info',
+  'javascript',
   'javascripts/.*',
+  'jserror/?',
   'locales\\.json',
+  'locales/.*',
+  'rest/.*',
   'pluginfw/.*',
+  'robots.txt',
   'static/.*',
+  'stats/?',
+  'tests/frontend(?:/.*)?'
 ].join('|')})$`);
 
 exports.normalizeAuthzLevel = (level) => {

src/node/server.js

@@ -46,6 +46,7 @@ const hooks = require('../static/js/pluginfw/hooks');
 const pluginDefs = require('../static/js/pluginfw/plugin_defs');
 const plugins = require('../static/js/pluginfw/plugins');
 const settings = require('./utils/Settings');
+const stats = require('./stats');
 
 const logger = log4js.getLogger('server');
 
@@ -104,8 +105,6 @@ exports.start = async () => {
     // Check if Etherpad version is up-to-date
     UpdateCheck.check();
 
-    // start up stats counting system
-    const stats = require('./stats');
     stats.gauge('memoryUsage', () => process.memoryUsage().rss);
     stats.gauge('memoryUsageHeap', () => process.memoryUsage().heapUsed);
 
@@ -215,6 +214,7 @@ exports.exit = async (err = null) => {
     logger.info('Received SIGTERM signal');
     err = null;
   } else if (err != null) {
+    logger.error(`Metrics at time of fatal error:\n${JSON.stringify(stats.toJSON(), null, 2)}`);
     logger.error(err.stack || err.toString());
     process.exitCode = 1;
     if (exitCalled) {