Merge branch 'release/v8.0.0' into develop

# Conflicts:
#	DocService/sources/databaseConnectors/baseConnector.js

Common/sources/commondefines.js

@@ -114,6 +114,7 @@ function InputCommand(data, copyExplicit) {
       this['wopiParams'] = data['wopiParams'];
       this['builderParams'] = data['builderParams'];
       this['userconnectiondocid'] = data['userconnectiondocid'];
+      this['originformat'] = data['originformat'];
     }
   } else {
     this['c'] = undefined;//string command
@@ -169,6 +170,7 @@ function InputCommand(data, copyExplicit) {
     this['status_info_in'] = undefined;
     this['attempt'] = undefined;
     this['convertToOrigin'] = undefined;
+    this['originformat'] = undefined;
   }
 }
 InputCommand.prototype = {
@@ -226,6 +228,12 @@ InputCommand.prototype = {
   setFormat: function(data) {
     this['format'] = data;
   },
+  getOriginFormat: function() {
+    return this['originformat'];
+  },
+  setOriginFormat: function(data) {
+    this['originformat'] = data;
+  },
   getUrl: function() {
     return this['url'];
   },

Common/sources/constants.js

@@ -113,6 +113,7 @@ exports.AVS_OFFICESTUDIO_FILE_DOCUMENT_HTML_IN_CONTAINER = exports.AVS_OFFICESTU
 exports.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX_PACKAGE = exports.AVS_OFFICESTUDIO_FILE_DOCUMENT + 0x0014;
 exports.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM = exports.AVS_OFFICESTUDIO_FILE_DOCUMENT + 0x0015;
 exports.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF = exports.AVS_OFFICESTUDIO_FILE_DOCUMENT + 0x0016;
+exports.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF = exports.AVS_OFFICESTUDIO_FILE_DOCUMENT + 0x0017;
 
 exports.AVS_OFFICESTUDIO_FILE_PRESENTATION = 0x0080;
 exports.AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX = exports.AVS_OFFICESTUDIO_FILE_PRESENTATION + 0x0001;

Common/sources/formatchecker.js

@@ -34,6 +34,7 @@
 
 var path = require('path');
 var constants = require('./constants');
+const {open} = require("node:fs/promises");
 
 function getImageFormatBySignature(buffer) {
   var length = buffer.length;
@@ -383,6 +384,8 @@ exports.getStringFromFormat = function(format) {
       return 'oform';
     case constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF:
       return 'docxf';
+    case constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF:
+      return 'pdf';
 
     case constants.AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX:
       return 'pptx';
@@ -534,6 +537,7 @@ exports.isOOXFormat = function(format) {
   || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOTM === format
   || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM === format
   || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF === format
+  || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF === format
   || constants.AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX === format
   || constants.AVS_OFFICESTUDIO_FILE_PRESENTATION_PPSX === format
   || constants.AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTM === format
@@ -544,4 +548,46 @@ exports.isOOXFormat = function(format) {
   || constants.AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLSM === format
   || constants.AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLTX === format
   || constants.AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLTM === format;
-};
+};
+function getDocumentFormatBySignature(buffer) {
+  if (!buffer) {
+    return constants.AVS_OFFICESTUDIO_FILE_UNKNOWN;
+  }
+  let text = buffer.toString("latin1");
+  // Check for binary DOCT format.
+  if (4 <= text.length && text[0] === 'D' && text[1] === 'O' && text[2] === 'C' && text[3] === 'Y') {
+    return constants.AVS_OFFICESTUDIO_FILE_CANVAS_WORD;
+  }
+
+  // Check for binary XLST format
+  if (4 <= text.length && text[0] === 'X' && text[1] === 'L' && text[2] === 'S' && text[3] === 'Y') {
+    return constants.AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET;
+  }
+
+  // Check for binary PPTT format
+  if (4 <= text.length && text[0] === 'P' && text[1] === 'P' && text[2] === 'T' && text[3] === 'Y') {
+    return constants.AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION;
+  }
+
+  // Unknown format
+  return constants.AVS_OFFICESTUDIO_FILE_UNKNOWN;
+};
+async function getDocumentFormatByFile(file) {
+  let firstBytesLen = 100;
+  let buffer;
+  let fd;
+  try {
+    fd = await open(file, 'r');
+    const stream = fd.createReadStream({ start: 0, end: firstBytesLen });
+    let chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(Buffer.from(chunk));
+    }
+    buffer = Buffer.concat(chunks);
+  } finally {
+    await fd?.close();
+  }
+  return getDocumentFormatBySignature(buffer);
+};
+exports.getDocumentFormatBySignature = getDocumentFormatBySignature;
+exports.getDocumentFormatByFile = getDocumentFormatByFile

Common/sources/utils.js

@@ -319,11 +319,11 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A
     if (!options.headers) {
       options.headers = {};
     }
+    if (opt_Authorization) {
+      options.headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization;
+    }
     if (opt_headers) {
       Object.assign(options.headers, opt_headers);
-      // options.headers = opt_headers;
-    } else if (opt_Authorization) {
-      options.headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization;
     }
     let fError = function(err) {
       reject(err);
@@ -356,7 +356,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A
       var contentLength = response.caseless.get('content-length');
       if (contentLength && (contentLength - 0) > sizeLimit) {
         raiseError(this, 'EMSGSIZE', 'Error response: content-length:' + contentLength);
-      } else if (response.statusCode !== 200) {
+      } else if (response.statusCode !== 200 && response.statusCode !== 206) {
         let code = response.statusCode;
         let responseHeaders = JSON.stringify(response.headers);
         let error = new Error(`Error response: statusCode:${code}; headers:${responseHeaders};`);

DocService/package.json

@@ -38,7 +38,7 @@
     "mysql2": "2.3.3",
     "oracledb": "6.0.1",
     "pg": "8.8.0",
-    "redis": "4.6.5",
+    "redis": "4.6.11",
     "retry": "0.12.0",
     "socket.io": "4.7.1",
     "underscore": "1.13.1",

DocService/sources/DocsCoServer.js

@@ -472,6 +472,14 @@ function addPresence(ctx, conn, updateCunters) {
     }
   });
 }
+async function updatePresence(ctx, conn) {
+  if (editorData.updatePresence) {
+    return await editorData.updatePresence(ctx, conn.docId, conn.user.id);
+  } else {
+    //todo remove if after 7.6. code for backward compatibility, because redis in separate repo
+    return await editorData.addPresence(ctx, conn.docId, conn.user.id, utils.getConnectionInfoStr(conn));
+  }
+}
 function removePresence(ctx, conn) {
   return co(function* () {
     yield editorData.removePresence(ctx, conn.docId, conn.user.id);
@@ -1732,6 +1740,7 @@ exports.install = function(server, callbackFunction) {
     const tenTokenEnableBrowser = ctx.getCfg('services.CoAuthoring.token.enable.browser', cfgTokenEnableBrowser);
     const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles);
 
+    ctx.logger.info("Connection closed or timed out: reason = %s", reason);
     var userLocks, reconnected = false, bHasEditors, bHasChanges;
     var docId = conn.docId;
     if (null == docId) {
@@ -1741,7 +1750,7 @@ exports.install = function(server, callbackFunction) {
     let participantsTimestamp;
     var tmpUser = conn.user;
     var isView = tmpUser.view;
-    ctx.logger.info("Connection closed or timed out: reason = %s", reason);
+
     var isCloseCoAuthoringTmp = conn.isCloseCoAuthoring;
     if (reason) {
       //Notify that participant has gone
@@ -2022,7 +2031,6 @@ exports.install = function(server, callbackFunction) {
   function* sendFileErrorAuth(ctx, conn, sessionId, errorId, code) {
     const tenTokenEnableBrowser = ctx.getCfg('services.CoAuthoring.token.enable.browser', cfgTokenEnableBrowser);
 
-    conn.isCloseCoAuthoring = true;
     conn.sessionId = sessionId;//restore old
     //Kill previous connections
     connections = _.reject(connections, function(el) {
@@ -3812,7 +3820,7 @@ exports.install = function(server, callbackFunction) {
           if (constants.CONN_CLOSED === conn.conn.readyState) {
             ctx.logger.error('expireDoc connection closed');
           }
-          yield addPresence(ctx, conn, false);
+          yield updatePresence(ctx, conn);
           if (utils.isLiveViewer(conn)) {
             countLiveViewByShard++;
             tenant.countLiveViewByShard++;
@@ -3848,6 +3856,7 @@ exports.install = function(server, callbackFunction) {
           let aggregationCtx = new operationContext.Context();
           aggregationCtx.init(tenantManager.getDefautTenant(), ctx.docId, ctx.userId);
           //yield ctx.initTenantCache();//no need
+          yield* collectStats(aggregationCtx, countEditByShard, countLiveViewByShard, countViewByShard);
           yield editorData.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard);
           yield editorData.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard);
           yield editorData.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard);

DocService/sources/canvasservice.js

@@ -316,6 +316,9 @@ function addPasswordToCmd(ctx, cmd, docPasswordStr) {
     cmd.setExternalChangeInfo(docPassword.change);
   }
 }
+function addOriginFormat(ctx, cmd, row) {
+  cmd.setOriginFormat(row && row.change_id);
+}
 
 function changeFormatByOrigin(ctx, row, format) {
   const tenAssemblyFormatAsOrigin = ctx.getCfg('services.CoAuthoring.server.assemblyFormatAsOrigin', cfgAssemblyFormatAsOrigin);
@@ -620,6 +623,7 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration,
   }
   yield* addRandomKeyTaskCmd(ctx, cmd);
   addPasswordToCmd(ctx, cmd, row.password);
+  addOriginFormat(ctx, cmd, row);
   let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback);
   cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback));
   cmd.setOutputFormat(changeFormatByOrigin(ctx, row, cmd.getOutputFormat()));
@@ -721,9 +725,6 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
           const filterPrivate = !authorizations[i] || !tenAllowPrivateIPAddressForSignedRequests;
           let getRes = yield utils.downloadUrlPromise(ctx, urlSource, tenImageDownloadTimeout, tenImageSize, authorizations[i], filterPrivate);
           data = getRes.body;
-
-          data = yield utilsDocService.fixImageExifRotation(ctx, data);
-
           urlParsed = urlModule.parse(urlSource);
         } catch (e) {
           data = undefined;
@@ -735,6 +736,9 @@ function* commandImgurls(ctx, conn, cmd, outputData) {
           }
         }
       }
+
+      data = yield utilsDocService.fixImageExifRotation(ctx, data);
+
       var outputUrl = {url: 'error', path: 'error'};
       if (data) {
         let format = formatChecker.getImageFormat(ctx, data);
@@ -1398,6 +1402,7 @@ exports.downloadAs = function(req, res) {
       if (!cmd.getWithoutPassword()) {
         addPasswordToCmd(ctx, cmd, row && row.password);
       }
+      addOriginFormat(ctx, cmd, row);
       cmd.setData(req.body);
       var outputData = new OutputData(cmd.getCommand());
       switch (cmd.getCommand()) {
@@ -1622,23 +1627,35 @@ exports.downloadFile = function(req, res) {
         res.sendStatus(filterStatus);
         return;
       }
+      let headers;
+      if (req.get('Range')) {
+        headers = {
+          'Range': req.get('Range')
+        }
+      }
+
       const filterPrivate = !authorization || !tenAllowPrivateIPAddressForSignedRequests;
-      yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, filterPrivate, null, res);
+      yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, filterPrivate, headers, res);
 
       if (clientStatsD) {
         clientStatsD.timing('coauth.downloadFile', new Date() - startDate);
       }
     }
     catch (err) {
       ctx.logger.error('Error downloadFile: %s', err.stack);
-      if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
-        res.sendStatus(408);
-      } else if (err.code === 'EMSGSIZE') {
-        res.sendStatus(413);
-      } else if (err.response) {
-        res.sendStatus(err.response.statusCode);
-      } else {
-        res.sendStatus(400);
+      //catch errors because status may be sent while piping to response
+      try {
+        if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
+          res.sendStatus(408);
+        } else if (err.code === 'EMSGSIZE') {
+          res.sendStatus(413);
+        } else if (err.response) {
+          res.sendStatus(err.response.statusCode);
+        } else {
+          res.sendStatus(400);
+        }
+      } catch (err) {
+        ctx.logger.error('Error downloadFile: %s', err.stack);
       }
     }
     finally {
@@ -1675,6 +1692,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId
         let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback);
         cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback));
         addPasswordToCmd(ctx, cmd, row && row.password);
+        addOriginFormat(ctx, cmd, row);
         yield* addRandomKeyTaskCmd(ctx, cmd);
         var queueData = getSaveTask(ctx, cmd);
         queueData.setFromChanges(true);

DocService/sources/editorDataMemory.js

@@ -96,6 +96,9 @@ EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken)
 EditorData.prototype.addPresence = function(ctx, docId, userId, userInfo) {
   return Promise.resolve();
 };
+EditorData.prototype.updatePresence = function(ctx, docId, userId) {
+  return Promise.resolve();
+};
 EditorData.prototype.removePresence = function(ctx, docId, userId) {
   return Promise.resolve();
 };