diff --git a/Common/sources/constants.js b/Common/sources/constants.js index 3e8e7aaaf..67d5a6047 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -50,6 +50,7 @@ exports.VIEWER_ONLY = /^(?:(pdf|djvu|xps|oxps))$/; exports.DEFAULT_DOC_ID = 'docId'; exports.DEFAULT_USER_ID = 'userId'; exports.ALLOWED_PROTO = /^https?$/i; +exports.SHARED_KEY_NAME = 'WOPISrc'; exports.RIGHTS = { None : 0, diff --git a/Common/sources/operationContext.js b/Common/sources/operationContext.js index ab2643a59..0eeb11a5f 100644 --- a/Common/sources/operationContext.js +++ b/Common/sources/operationContext.js @@ -41,17 +41,18 @@ function Context(){ this.logger = logger.getLogger('nodeJS'); this.initDefault(); } -Context.prototype.init = function(tenant, docId, userId) { +Context.prototype.init = function(tenant, docId, userId, opt_shardKey) { this.setTenant(tenant); this.setDocId(docId); this.setUserId(userId); + this.setShardKey(opt_shardKey); this.config = null; this.secret = null; this.license = null; }; Context.prototype.initDefault = function() { - this.init(tenantManager.getDefautTenant(), constants.DEFAULT_DOC_ID, constants.DEFAULT_USER_ID); + this.init(tenantManager.getDefautTenant(), constants.DEFAULT_DOC_ID, constants.DEFAULT_USER_ID, undefined); }; Context.prototype.initFromConnection = function(conn) { let tenant = tenantManager.getTenantByConnection(this, conn); @@ -64,19 +65,21 @@ Context.prototype.initFromConnection = function(conn) { } } let userId = conn.user?.id; - this.init(tenant, docId || this.docId, userId || this.userId); + let shardKey = utils.getShardByConnection(this, conn); + this.init(tenant, docId || this.docId, userId || this.userId, shardKey); }; Context.prototype.initFromRequest = function(req) { let tenant = tenantManager.getTenantByRequest(this, req); - this.init(tenant, this.docId, this.userId); + let shardKey = utils.getShardKeyByRequest(this, req); + this.init(tenant, this.docId, this.userId, shardKey); }; Context.prototype.initFromTaskQueueData = function(task) { let ctx = task.getCtx(); - this.init(ctx.tenant, ctx.docId, ctx.userId); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); }; Context.prototype.initFromPubSub = function(data) { let ctx = data.ctx; - this.init(ctx.tenant, ctx.docId, ctx.userId); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); }; Context.prototype.initTenantCache = async function() { this.config = await tenantManager.getTenantConfig(this); @@ -95,11 +98,15 @@ Context.prototype.setUserId = function(userId) { this.userId = userId; this.logger.addContext('USERID', userId); }; +Context.prototype.setShardKey = function(shardKey) { + this.shardKey = shardKey; +}; Context.prototype.toJSON = function() { return { tenant: this.tenant, docId: this.docId, - userId: this.userId + userId: this.userId, + shardKey: this.shardKey } }; Context.prototype.getCfg = function(property, defaultValue) { diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index d4aad1fc6..9b2b2ecfa 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -39,6 +39,7 @@ var utils = require("./utils"); var crypto = require('crypto'); const ms = require('ms'); const commonDefines = require('./../../Common/sources/commondefines'); +const constants = require('./../../Common/sources/constants'); var config = require('config'); var configStorage = config.get('storage'); @@ -141,6 +142,9 @@ async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_cre url += '?md5=' + encodeURIComponent(md5); url += '&expires=' + encodeURIComponent(expires); + if (ctx.shardKey) { + url += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } url += '&filename=' + userFriendlyName; return url; } diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 3faef68e8..8f61532f2 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -731,6 +731,14 @@ function getDomainByRequest(ctx, req) { } exports.getDomainByConnection = getDomainByConnection; exports.getDomainByRequest = getDomainByRequest; +function getShardByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARED_KEY_NAME]; +} +function getShardKeyByRequest(ctx, req) { + return req.query[constants.SHARED_KEY_NAME]; +} +exports.getShardByConnection = getShardByConnection; +exports.getShardKeyByRequest = getShardKeyByRequest; function stream2Buffer(stream) { return new Promise(function(resolve, reject) { if (!stream.readable) { diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index c6ae466a1..00b5cf33d 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -902,7 +902,7 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti } return res; } -async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn) { +async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn, opt_initShardKey) { ctx.logger.debug('startForceSave start'); let res = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; let startedForceSave; @@ -967,7 +967,7 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ //start new convert let status = await converterService.convertFromChanges(ctx, docId, baseUrl, forceSave, startedForceSave.changeInfo, opt_userdata, opt_formdata, opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, priority, expiration, - opt_queue); + opt_queue, undefined, opt_initShardKey); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); } else { @@ -1311,7 +1311,7 @@ function* cleanDocumentOnExitNoChanges(ctx, docId, opt_userId, opt_userIndex, op yield* cleanDocumentOnExit(ctx, docId, false, opt_userIndex); } -function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay) { +function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay, opt_initShardKey) { return co(function*(){ const tenAscSaveTimeOutDelay = ctx.getCfg('services.CoAuthoring.server.savetimeoutdelay', cfgAscSaveTimeOutDelay); @@ -1329,7 +1329,7 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_n } while (true) { if (!sqlBase.isLockCriticalSection(docId)) { - canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue); + canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue, opt_initShardKey); break; } yield utils.sleep(c_oAscLockTimeOutDelay); @@ -2496,7 +2496,8 @@ exports.install = function(server, callbackFunction) { let format = data.openCmd && data.openCmd.format; upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), data.documentCallbackUrl, format); curIndexUser = upsertRes.insertId; - if (upsertRes.isInsert && undefined !== data.timezoneOffset) { + //todo update additional in commandOpenStartPromise + if ((upsertRes.isInsert || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey)) { //todo insert in commandOpenStartPromise. insert here for database compatibility if (false === canvasService.hasAdditionalCol) { let selectRes = yield taskResult.select(ctx, docId); @@ -2506,8 +2507,13 @@ exports.install = function(server, callbackFunction) { let task = new taskResult.TaskResultData(); task.tenant = ctx.tenant; task.key = docId; - //todo duplicate created_at because CURRENT_TIMESTAMP uses server timezone - task.additional = sqlBase.DocumentAdditional.prototype.setOpenedAt(Date.now(), data.timezoneOffset); + if (undefined !== data.timezoneOffset) { + //todo duplicate created_at because CURRENT_TIMESTAMP uses server timezone + task.additional = sqlBase.DocumentAdditional.prototype.setOpenedAt(Date.now(), data.timezoneOffset); + } + if (ctx.shardKey) { + task.additional += sqlBase.DocumentAdditional.prototype.setShardKey(ctx.shardKey); + } yield taskResult.update(ctx, task); } else { ctx.logger.warn('auth unknown column "additional"'); diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 3f2a486fc..674306515 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -609,12 +609,15 @@ function* commandSendMailMerge(ctx, cmd, outputData) { outputData.setData(cmd.getSaveKey()); } } -let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, opt_queue) { +let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey) { var selectRes = yield taskResult.select(ctx, cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; if (!row) { return; } + if (opt_initShardKey) { + ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + } yield* addRandomKeyTaskCmd(ctx, cmd); addPasswordToCmd(ctx, cmd, row.password); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); @@ -1502,8 +1505,13 @@ function getPrintFileUrl(ctx, docId, baseUrl, filename) { } //while save printed file Chrome's extension seems to rely on the resource name set in the URI https://stackoverflow.com/a/53593453 //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path - var userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); - return `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}&filename=${userFriendlyName}`; + let userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); + let res = `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}`; + if (ctx.shardKey) { + res += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + res += `&filename=${userFriendlyName}`; + return res; }); } exports.getPrintFileUrl = getPrintFileUrl; @@ -1638,7 +1646,7 @@ exports.downloadFile = function(req, res) { } }); }; -exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue) { +exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue, opt_initShardKey) { return co(function* () { try { var startDate = null; @@ -1652,7 +1660,10 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId if (row && row.status == commonDefines.FileStatus.SaveVersion && row.status_info == statusInfo) { if (null == optFormat) { optFormat = changeFormatByOrigin(ctx, row, constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML); - } + } + if (opt_initShardKey) { + ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); cmd.setDocId(docId); diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index cbe6f5113..6ee74f549 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -183,7 +183,7 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, } async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, opt_userConnectionId, - opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey) { + opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey, opt_initShardKey) { var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfcm'); cmd.setDocId(docId); @@ -213,7 +213,7 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange cmd.setRedisKey(opt_redisKey); } - await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue); + await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); var fileTo = constants.OUTPUT_NAME; let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat()); if (outputExt) { diff --git a/DocService/sources/databaseConnectors/connectorUtilities.js b/DocService/sources/databaseConnectors/connectorUtilities.js index 89f6db330..b7a988c4c 100644 --- a/DocService/sources/databaseConnectors/connectorUtilities.js +++ b/DocService/sources/databaseConnectors/connectorUtilities.js @@ -151,6 +151,23 @@ DocumentAdditional.prototype.getOpenedAt = function(str) { return res; }; +DocumentAdditional.prototype.setShardKey = function(shardKey) { + let additional = new DocumentAdditional(); + additional.data.push({shardKey}); + return additional.toSQLInsert(); +}; +DocumentAdditional.prototype.getShardKey = function(str) { + let res; + let val = new DocumentAdditional(); + val.fromString(str); + val.data.forEach((elem) => { + if (elem.shardKey) { + res = elem.shardKey; + } + }); + return res; +}; + module.exports = { UserCallback, DocumentPassword, diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index 6690dd68d..496e6a934 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -47,6 +47,7 @@ var commondefines = require('./../../Common/sources/commondefines'); var queueService = require('./../../Common/sources/taskqueueRabbitMQ'); var operationContext = require('./../../Common/sources/operationContext'); var pubsubService = require('./pubsubRabbitMQ'); +const sqlBase = require("./databaseConnectors/baseConnector"); var cfgExpFilesCron = config.get('services.CoAuthoring.expire.filesCron'); var cfgExpDocumentsCron = config.get('services.CoAuthoring.expire.documentsCron'); @@ -76,7 +77,8 @@ var checkFileExpire = function(expireSeconds) { for (var i = 0; i < expired.length; ++i) { let tenant = expired[i].tenant; let docId = expired[i].id; - ctx.init(tenant, docId, ctx.userId); + let shardKey = sqlBase.DocumentAdditional.prototype.getShardKey(expired[i].additional); + ctx.init(tenant, docId, ctx.userId, shardKey); yield ctx.initTenantCache(); //todo tenant //check that no one is in the document @@ -122,7 +124,8 @@ var checkDocumentExpire = function() { yield ctx.initTenantCache(); var hasChanges = yield docsCoServer.hasChanges(ctx, docId); if (hasChanges) { - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true); + //todo opt_initShardKey from getDocumentPresenceExpired data or from db + yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true, true); startSaveCount++; } else { yield docsCoServer.cleanDocumentOnExitNoChangesPromise(ctx, docId); @@ -170,9 +173,10 @@ let forceSaveTimeout = function() { if (docId) { ctx.init(tenant, docId, ctx.userId); yield ctx.initTenantCache(); + //todo opt_initShardKey from ForceSave data or from db actions.push(docsCoServer.startForceSave(ctx, docId, commondefines.c_oAscForceSaveTypes.Timeout, undefined, undefined, undefined, undefined, - undefined, undefined, undefined, undefined, queue, pubsub)); + undefined, undefined, undefined, undefined, queue, pubsub, undefined, true)); } } yield Promise.all(actions);