From 97457e145551a23bfd65a270f7fc57a5cedd9e22 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Sat, 11 Mar 2023 14:09:44 +0300 Subject: [PATCH 01/31] minor adjustment in rebuildDatabase --- ClusterOperator/Backlog.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ClusterOperator/Backlog.js b/ClusterOperator/Backlog.js index 0295071..54c93b2 100644 --- a/ClusterOperator/Backlog.js +++ b/ClusterOperator/Backlog.js @@ -299,7 +299,7 @@ class BackLog { await this.BLClient.createDB(config.dbInitDB); this.UserDBClient.setDB(config.dbInitDB); await this.BLClient.setDB(config.dbBacklog); - const records = await this.BLClient.execute('SELECT * FROM backlog WHERE seq=? ORDER BY seq', [seqNo]); + await this.BLClient.execute('DELETE FROM backlog WHERE seq>? ORDER BY seq', [seqNo]); } } catch (e) { log.error(e); From 62ab1f2f9cb1f2722dbf84539753912b5267c0aa Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Sat, 11 Mar 2023 15:02:31 +0300 Subject: [PATCH 02/31] time machine rollback api --- ClusterOperator/Backlog.js | 5 +++-- ClusterOperator/Operator.js | 35 +++++++++++++++++++++++++++++++++++ ClusterOperator/server.js | 9 +++++++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/ClusterOperator/Backlog.js b/ClusterOperator/Backlog.js index 54c93b2..4443f36 100644 --- a/ClusterOperator/Backlog.js +++ b/ClusterOperator/Backlog.js @@ -311,13 +311,14 @@ class BackLog { } // eslint-disable-next-line no-await-in-loop } - await this.BLClient.execute('DELETE FROM backlog WHERE seq>? ORDER BY seq', [seqNo]); + await this.BLClient.execute('DELETE FROM backlog WHERE seq>?', [seqNo]); + await this.clearBuffer(); } } catch (e) { log.error(e); } this.buffer = []; - log.info('All buffer data removed successfully.'); + log.info(`DB and backlog rolled back to ${seqNo}`); } /** diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index 7fa0b53..22b5a37 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -213,6 +213,18 @@ class Operator { await BackLog.pushKey(decKey, value); Operator.keys[decKey] = value; }); + this.masterWSConn.on('rollBack', async (seqNo) => { + if (this.status === 'SYNC') { + this.status = 'ROLLBACK'; + await BackLog.rollBack(seqNo); + this.syncLocalDB(); + } else { + const tempStatus = this.status; + this.status = 'ROLLBACK'; + await BackLog.rollBack(seqNo); + this.status = tempStatus; + } + }); } catch (e) { log.error(e); this.masterWSConn.removeAllListeners(); @@ -322,6 +334,25 @@ class Operator { return null; } + /** + * [rollBack] + * @param {int} seq [description] + */ + static async rollBack(seqNo) { + if (this.IamMaster) { + this.status = 'ROLLBACK'; + log.info(`rooling back to ${seqNo}`); + this.serverSocket.emit('rollback', seqNo); + await BackLog.rebuildDatabase(seqNo); + this.status = 'OK'; + } else { + const { masterWSConn } = this; + masterWSConn.emit('rollBack', seqNo, (response) => { + log.info(response.result); + }); + } + } + /** * [setServerSocket] * @param {socket} socket [description] @@ -471,6 +502,10 @@ class Operator { // log.info(JSON.stringify(response.records)); BackLog.executeLogs = false; for (const record of response.records) { + if (this.status !== 'SYNC') { + log.warn('Sync proccess halted.', 'red'); + return; + } await BackLog.pushQuery(record.query, record.seq, record.timestamp); } if (BackLog.bufferStartSequenceNumber > 0 && BackLog.bufferStartSequenceNumber <= BackLog.sequenceNumber) copyBuffer = true; diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index ea612d8..4504c0e 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -77,8 +77,7 @@ function startUI() { const { seqNo } = req.query; if (whiteList.length && seqNo) { if (whiteList.includes(remoteIp)) { - log.info(`rooling back to ${seqNo}`); - BackLog.rebuildDatabase(seqNo); + Operator.rollBack(seqNo); } } }); @@ -313,6 +312,12 @@ async function initServer() { } callback({ status: Operator.status }); }); + socket.on('rollBack', async (seqNo, callback) => { + if (Operator.IamMaster) { + Operator.rollBack(seqNo); + } + callback({ status: Operator.status }); + }); } else { // log.info(`rejected from ${ip}`); socket.disconnect(); From 18a265f168711b4ad2638bb16ba409a08a2ada2f Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Sun, 19 Mar 2023 10:36:03 +0300 Subject: [PATCH 03/31] Add team access for debug & support --- ClusterOperator/Operator.js | 2 +- ClusterOperator/server.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index 22b5a37..911645c 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -283,7 +283,7 @@ class Operator { if (this.authorizedApp === null) this.authorizedApp = remoteIp; const whiteList = config.whiteListedIps.split(','); // temporary whitelist ip for flux team debugging, should be removed after final release - if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '167.235.234.45') { + if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '206.79.215.43') { return true; } if (!this.operator.IamMaster && (config.AppName.includes('wordpress') || config.authMasterOnly)) return false; diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 4504c0e..e3e3998 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -51,7 +51,7 @@ function startUI() { } if (whiteList.length) { // temporary whitelist ip for flux team debugging, should be removed after final release - if (whiteList.includes(remoteIp) || remoteIp === '167.235.234.45') { + if (whiteList.includes(remoteIp) || remoteIp === '206.79.215.43') { res.send(` + + + +
+
+

Time Machine

+

Adjust the slider to choose a date, followed by selecting 'Query' from the backlog.

+
+
+
+
+
+
+
+
+
+
+
+
+
+ + + From 7f7268f349eadd9f30e348b7ac26534a87acffbd Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 18 Apr 2023 14:40:37 +0300 Subject: [PATCH 09/31] feat: Time machine --- ClusterOperator/Backlog.js | 47 +++++- ClusterOperator/Operator.js | 4 +- ClusterOperator/server.js | 52 ++++++- ui/index.html | 276 ++++++++++++++++++++++++++++++++---- 4 files changed, 340 insertions(+), 39 deletions(-) diff --git a/ClusterOperator/Backlog.js b/ClusterOperator/Backlog.js index 4443f36..a3ebed7 100644 --- a/ClusterOperator/Backlog.js +++ b/ClusterOperator/Backlog.js @@ -188,6 +188,28 @@ class BackLog { return []; } + /** + * [getLogsByTime] + * @param {int} startFrom [description] + * @param {int} length [description] + * @return {Array} + */ + static async getLogsByTime(startFrom, length) { + if (!this.BLClient) { + log.error('Backlog not created yet. Call createBacklog() first.'); + return []; + } + try { + if (config.dbType === 'mysql') { + const totalRecords = await this.BLClient.execute(`SELECT seq, LEFT(query,200) as query, timestamp FROM ${config.dbBacklogCollection} WHERE timestamp >= ? AND timestamp < ? ORDER BY seq`, [startFrom, Number(startFrom) + Number(length)]); + return totalRecords; + } + } catch (e) { + log.error(e); + } + return []; + } + /** * [getLogs] * @param {int} index [description] @@ -201,8 +223,29 @@ class BackLog { try { if (config.dbType === 'mysql') { const record = await this.BLClient.execute(`SELECT * FROM ${config.dbBacklogCollection} WHERE seq=?`, [index]); - // log.info(`backlog records ${startFrom},${pageSize}:${JSON.stringify(totalRecords)}`); - return record; + log.info(record); + return record[0]; + } + } catch (e) { + log.error(e); + } + return []; + } + + /** + * [getDateRange] + * @return {object} + */ + static async getDateRange() { + if (!this.BLClient) { + log.error('Backlog not created yet. Call createBacklog() first.'); + return []; + } + try { + if (config.dbType === 'mysql') { + const record = await this.BLClient.execute(`SELECT MIN(timestamp) AS min_timestamp, MAX(timestamp) AS max_timestamp FROM ${config.dbBacklogCollection}`); + log.info(record); + return record[0]; } } catch (e) { log.error(e); diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index ae495eb..3ec1d57 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -224,12 +224,12 @@ class Operator { this.masterWSConn.on('rollBack', async (seqNo) => { if (this.status === 'SYNC') { this.status = 'ROLLBACK'; - await BackLog.rollBack(seqNo); + await BackLog.rebuildDatabase(seqNo); this.syncLocalDB(); } else { const tempStatus = this.status; this.status = 'ROLLBACK'; - await BackLog.rollBack(seqNo); + await BackLog.rebuildDatabase(seqNo); this.status = tempStatus; } }); diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 486a22d..e7047f2 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -25,6 +25,7 @@ function startUI() { const app = express(); app.use(cors()); app.use(bodyParser.json()); + app.use(bodyParser.urlencoded({ extended: false })); fs.writeFileSync('errors.txt', `version: ${config.version}
`); fs.writeFileSync('warnings.txt', `version: ${config.version}
`); fs.writeFileSync('info.txt', `version: ${config.version}
`); @@ -72,13 +73,50 @@ function startUI() { } }); - app.get('/rollback', (req, res) => { + app.post('/rollback', async (req, res) => { const remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); - const { seqNo } = req.query; + let { seqNo } = req.body; + seqNo = seqNo || req.query.seqNo; + console.log(req.body); + console.log(seqNo); if (whiteList.length && seqNo) { if (whiteList.includes(remoteIp)) { - Operator.rollBack(seqNo); + console.log(seqNo); + await Operator.rollBack(seqNo); + res.send({ status: 'OK' }); + } + } + }); + + app.get('/stats', (req, res) => { + const remoteIp = utill.convertIP(req.ip); + const whiteList = config.whiteListedIps.split(','); + if (whiteList.length) { + if (whiteList.includes(remoteIp)) { + res.send(Operator.OpNodes); + } + } + }); + + app.get('/getLogDateRange', async (req, res) => { + const remoteIp = utill.convertIP(req.ip); + const whiteList = config.whiteListedIps.split(','); + if (whiteList.length) { + if (whiteList.includes(remoteIp)) { + res.send(await BackLog.getDateRange()); + } + } + }); + + app.get('/getLogsByTime', async (req, res) => { + const remoteIp = utill.convertIP(req.ip); + const whiteList = config.whiteListedIps.split(','); + const { starttime } = req.query; + const { length } = req.query; + if (whiteList.length) { + if (whiteList.includes(remoteIp)) { + res.send(await BackLog.getLogsByTime(starttime, length)); } } }); @@ -159,9 +197,9 @@ function startUI() { console.log(remoteIp); console.log(whiteList); console.log(whiteList.includes(remoteIp)); - //if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '206.79.215.43') { + if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '206.79.215.43') { res.sendFile(path.join(__dirname, '../ui/index.html')); - //} + } }); app.listen(config.debugUIPort, () => { @@ -349,10 +387,10 @@ async function initServer() { }); */ log.info(`Api Server started on port ${config.apiPort}`); - // await Operator.findMaster(); + await Operator.findMaster(); log.info(`find master finished, master is ${Operator.masterNode}`); if (!Operator.IamMaster) { - // Operator.initMasterConnection(); + Operator.initMasterConnection(); } setInterval(async () => { Operator.doHealthCheck(); diff --git a/ui/index.html b/ui/index.html index 4e8b031..1eacec5 100644 --- a/ui/index.html +++ b/ui/index.html @@ -8,15 +8,58 @@ - - - @@ -30,17 +73,17 @@ -
-
-

Time Machine

-

Adjust the slider to choose a date, followed by selecting 'Query' from the backlog.

-
+ +
+
+
+

Time Machine

+
+
-
-
+
+

1. Choose Date and Time

+
+
+
+

2. Choose Rewind Point

+
+
+
+
+

3. Rewind

+ +

Caution! this action is irreversible and will permanently rewind your database to the selected datetime.

+
+ + Rewind in proggress, Please wait... +
+
+ Rewind Finished +
-
+
+
+
+

Dashboard

+
+
+ + + From ec2d0c4ef844c16374eea84b4b65a23ba1ac13e8 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Wed, 19 Apr 2023 18:51:47 +0300 Subject: [PATCH 10/31] timemachine UI improvements --- ClusterOperator/Backlog.js | 2 +- ClusterOperator/Operator.js | 25 ++++++++++++--------- ui/index.html | 45 +++++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/ClusterOperator/Backlog.js b/ClusterOperator/Backlog.js index a3ebed7..81d5673 100644 --- a/ClusterOperator/Backlog.js +++ b/ClusterOperator/Backlog.js @@ -201,7 +201,7 @@ class BackLog { } try { if (config.dbType === 'mysql') { - const totalRecords = await this.BLClient.execute(`SELECT seq, LEFT(query,200) as query, timestamp FROM ${config.dbBacklogCollection} WHERE timestamp >= ? AND timestamp < ? ORDER BY seq`, [startFrom, Number(startFrom) + Number(length)]); + const totalRecords = await this.BLClient.execute(`SELECT seq, LEFT(query,10) as query, timestamp FROM ${config.dbBacklogCollection} WHERE timestamp >= ? AND timestamp < ? ORDER BY seq`, [startFrom, Number(startFrom) + Number(length)]); return totalRecords; } } catch (e) { diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index 3ec1d57..88bb95b 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -222,6 +222,7 @@ class Operator { Operator.keys[decKey] = value; }); this.masterWSConn.on('rollBack', async (seqNo) => { + log.info(`rollback request from master, rewinding to ${seqNo}`); if (this.status === 'SYNC') { this.status = 'ROLLBACK'; await BackLog.rebuildDatabase(seqNo); @@ -347,17 +348,19 @@ class Operator { * @param {int} seq [description] */ static async rollBack(seqNo) { - if (this.IamMaster) { - this.status = 'ROLLBACK'; - log.info(`rooling back to ${seqNo}`); - this.serverSocket.emit('rollback', seqNo); - await BackLog.rebuildDatabase(seqNo); - this.status = 'OK'; - } else { - const { masterWSConn } = this; - masterWSConn.emit('rollBack', seqNo, (response) => { - log.info(response.result); - }); + if (this.status !== 'ROLLBACK') { + if (this.IamMaster) { + this.status = 'ROLLBACK'; + log.info(`rolling back to ${seqNo}`); + this.serverSocket.emit('rollback', seqNo); + await BackLog.rebuildDatabase(seqNo); + this.status = 'OK'; + } else { + const { masterWSConn } = this; + masterWSConn.emit('rollBack', seqNo, (response) => { + log.info(response.result); + }); + } } } diff --git a/ui/index.html b/ui/index.html index 1eacec5..7ccf979 100644 --- a/ui/index.html +++ b/ui/index.html @@ -36,20 +36,28 @@ cursor: pointer; border-top: solid 1px #e3e3e3; } + .query-h { + width: 100%; + height: 40px; + display: flex; + align-items: center; + background-color: #efefef; + padding: 0px 15px 0px 15px; + } .selected { background-color: #525252; color: white; } .q-date { - flex-shrink: 0; - margin-left: 10px; + flex-grow: 1; white-space: nowrap; } .q-text { - flex-grow: 1; + flex-shrink: 0; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; + margin-left: 10px; } .fs-7 { font-size: 12px; @@ -80,7 +88,7 @@ DB
    -
  • Time Machine
    • +
  • Point-in-time Recovery
  • Backup & Restore
  • Backlog Editor
  • Run Query
    • @@ -107,7 +115,7 @@
    -

    Time Machine

    +

    Point-in-time Recovery

    @@ -222,27 +230,30 @@

    Dashboard

    let startDate = new Date(data.min_timestamp); let endDate = new Date(data.max_timestamp); let dp = $('.datepicker-tm').datepicker({ - timepicker: true, language: 'en', inline: true, minDate: startDate, maxDate: endDate, - hoursStep: 4, - maxMinutes: 0, onSelect: function (date , formattedDate, datepicker) { let d = new Date(date); - console.log(d.getTimezoneOffset() ); - var timestamp = Math.floor(d.getTime() + d.getTimezoneOffset() * 60000); - console.log(timestamp); + //console.log(d.getTimezoneOffset() ); + var timestamp = Math.floor(d.getTime()); $.ajax({ - type: 'GET', dataType:"json", url: 'http://' + window.apiURL + `/getLogsByTime?starttime=${timestamp}&length=14400000`, + type: 'GET', dataType:"json", url: 'http://' + window.apiURL + `/getLogsByTime?starttime=${timestamp}&length=86400000`, success: function (data, status, xhr) { - console.log('data: ', data); - $('.queries-tm').html(''); - for (const record of data) { - let date = new Date(record.timestamp); - $('.queries-tm').append(`
    ${escapeHtml(record.query)}${date.toLocaleString()}
    `) + // console.log('data: ', data); + let points = 0; + $('.queries-tm').html(`
    Total sessions foundNumber of operations
    `); + for (let i = 0; i < data.length; i+=1) { + if (data[i].query.toLowerCase().startsWith('set sess')){ + points +=1; + let ops = 1; + while (i + 1 < data.length && !data[i + 1].query.toLowerCase().startsWith('set sess')) { i += 1; ops +=1 } + let date = new Date(data[i].timestamp); + $('.queries-tm').append(`
    ${data[i].seq}${date.toLocaleString()}${ops}
    `) + } } + $('#total-records').html(points); } }); From 50f5e4fad781765e0df3a48f046014feb6069b38 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Wed, 19 Apr 2023 19:18:40 +0300 Subject: [PATCH 11/31] minor improvements --- ui/index.html | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ui/index.html b/ui/index.html index 7ccf979..f67bce9 100644 --- a/ui/index.html +++ b/ui/index.html @@ -23,6 +23,7 @@ background-color: #fff; } .query { + font-size: 15px; height: 30px; width: 100%; display: flex; @@ -49,21 +50,20 @@ color: white; } .q-date { - flex-grow: 1; + flex-shrink: 0; white-space: nowrap; } .q-text { - flex-shrink: 0; + flex-grow: 1; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; - margin-left: 10px; } .fs-7 { font-size: 12px; } .queries-tm { - height: 600px; + height: 700px; overflow-y: scroll; } .datepicker--cell.-selected-, .datepicker--cell.-selected-.-current-,.datepicker--cell.-selected-.-focus- { @@ -243,14 +243,14 @@

    Dashboard

    success: function (data, status, xhr) { // console.log('data: ', data); let points = 0; - $('.queries-tm').html(`
    Total sessions foundNumber of operations
    `); + $('.queries-tm').html(`
    Total sessions foundDate and Time
    `); for (let i = 0; i < data.length; i+=1) { if (data[i].query.toLowerCase().startsWith('set sess')){ points +=1; let ops = 1; while (i + 1 < data.length && !data[i + 1].query.toLowerCase().startsWith('set sess')) { i += 1; ops +=1 } let date = new Date(data[i].timestamp); - $('.queries-tm').append(`
    ${data[i].seq}${date.toLocaleString()}${ops}
    `) + $('.queries-tm').append(`
    ${data[i].seq}${ops} Ops - ${date.toLocaleString()}
    `) } } $('#total-records').html(points); @@ -301,6 +301,10 @@

    Dashboard

    console.log(seq); document.getElementById("tm-rewind").setAttribute("data-seq", seq); } + function dn(e) { + e.preventDefault(); + e.stopPropagation(); + } function rewind(seq) { document.getElementById("tm-rewind").disabled = true; From ef27dbdf2a79b0513cfd985269eeb1c20efca75d Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Wed, 19 Apr 2023 19:31:08 +0300 Subject: [PATCH 12/31] remove extra logs --- ClusterOperator/server.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index e7047f2..2da31b5 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -194,9 +194,6 @@ function startUI() { app.get('/', (req, res) => { const remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); - console.log(remoteIp); - console.log(whiteList); - console.log(whiteList.includes(remoteIp)); if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '206.79.215.43') { res.sendFile(path.join(__dirname, '../ui/index.html')); } From ebb7c15bd3a9af09a25faa8a3fe0ad9f8c1f4c26 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Wed, 19 Apr 2023 19:58:21 +0300 Subject: [PATCH 13/31] minor fix --- ClusterOperator/Operator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index 88bb95b..38e7fca 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -352,7 +352,7 @@ class Operator { if (this.IamMaster) { this.status = 'ROLLBACK'; log.info(`rolling back to ${seqNo}`); - this.serverSocket.emit('rollback', seqNo); + this.serverSocket.emit('rollBack', seqNo); await BackLog.rebuildDatabase(seqNo); this.status = 'OK'; } else { From 9d134cbd28e7ac3a4790cd0069668c662530e46e Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Wed, 19 Apr 2023 20:12:13 +0300 Subject: [PATCH 14/31] fix log error --- ClusterOperator/Operator.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ClusterOperator/Operator.js b/ClusterOperator/Operator.js index 38e7fca..c5c87b7 100644 --- a/ClusterOperator/Operator.js +++ b/ClusterOperator/Operator.js @@ -357,11 +357,14 @@ class Operator { this.status = 'OK'; } else { const { masterWSConn } = this; - masterWSConn.emit('rollBack', seqNo, (response) => { - log.info(response.result); + return new Promise((resolve) => { + masterWSConn.emit('rollBack', seqNo, (response) => { + resolve(response.result); + }); }); } } + return null; } /** From efb7713d81d2f02a4555310a67bc1840c562f6e3 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 20 Apr 2023 15:43:55 +0300 Subject: [PATCH 15/31] fix wording --- .github/workflows/docker-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d0b7b3d..c6b9aa4 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -5,6 +5,8 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] + schedule: + - cron: "0 0 * * *" jobs: From 98eaf1c09338c85fbdfa8e7048caf727558e5669 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 20 Apr 2023 15:44:14 +0300 Subject: [PATCH 16/31] fix wording --- ui/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/index.html b/ui/index.html index f67bce9..d903357 100644 --- a/ui/index.html +++ b/ui/index.html @@ -122,7 +122,7 @@

    Point-in-time Recovery

    -

    1. Choose Date and Time

    +

    1. Choose a Date

    @@ -133,7 +133,7 @@

    Point-in-time Recovery

    3. Rewind

    -

    Caution! this action is irreversible and will permanently rewind your database to the selected datetime.

    +

    Caution! this action is irreversible and will permanently rewind your database to the selected date-time. All rewind points after the selected date will be removed.

    -

    2. Choose Rewind Point

    +

    2. Choose Recovery Point

    From 456601629157aa201bb8712362b6b46742258e0e Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 10:51:44 +0300 Subject: [PATCH 19/31] Fix: FDM health check for UI --- ClusterOperator/server.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 2da31b5..8812fa8 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -196,6 +196,8 @@ function startUI() { const whiteList = config.whiteListedIps.split(','); if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '206.79.215.43') { res.sendFile(path.join(__dirname, '../ui/index.html')); + } else { + res.send('Permission Denied.'); } }); From f7e31401c8e629457ce846e7b0e66ae5b185f9ce Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 10:58:49 +0300 Subject: [PATCH 20/31] remove team IP access from UI --- ClusterOperator/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 8812fa8..0462d77 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -194,7 +194,7 @@ function startUI() { app.get('/', (req, res) => { const remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); - if ((whiteList.length && whiteList.includes(remoteIp)) || remoteIp === '206.79.215.43') { + if ((whiteList.length && whiteList.includes(remoteIp))) { res.sendFile(path.join(__dirname, '../ui/index.html')); } else { res.send('Permission Denied.'); From 815345fe71e94d72f2b0acc06721befb1af8ee47 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 13:07:48 +0300 Subject: [PATCH 21/31] test headers --- ClusterOperator/server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 0462d77..10168c5 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -194,6 +194,7 @@ function startUI() { app.get('/', (req, res) => { const remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); + log.info(JSON.stringify(req.headers)); if ((whiteList.length && whiteList.includes(remoteIp))) { res.sendFile(path.join(__dirname, '../ui/index.html')); } else { From 2dce481d5ca3cce4e62004d449b14b17026f58e0 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 13:09:08 +0300 Subject: [PATCH 22/31] log UI access --- ClusterOperator/server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 10168c5..6c9159d 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -195,6 +195,7 @@ function startUI() { const remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); log.info(JSON.stringify(req.headers)); + log.info(`UI access from ${remoteIp}`); if ((whiteList.length && whiteList.includes(remoteIp))) { res.sendFile(path.join(__dirname, '../ui/index.html')); } else { From 1069b88abc0478d9f81b2a39b628605b7a486de1 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 13:16:13 +0300 Subject: [PATCH 23/31] check x-forwarded-for header --- ClusterOperator/server.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 6c9159d..a046dec 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -192,8 +192,11 @@ function startUI() { }); app.get('/', (req, res) => { - const remoteIp = utill.convertIP(req.ip); + let remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); + if (req.headers['x-forwarded-for']) { + remoteIp = req.headers['x-forwarded-for']; + } log.info(JSON.stringify(req.headers)); log.info(`UI access from ${remoteIp}`); if ((whiteList.length && whiteList.includes(remoteIp))) { From dfc526487dc96c56766a1416b30790201b60e476 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 13:20:51 +0300 Subject: [PATCH 24/31] remove extra logs --- ClusterOperator/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index a046dec..074fb48 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -197,8 +197,8 @@ function startUI() { if (req.headers['x-forwarded-for']) { remoteIp = req.headers['x-forwarded-for']; } - log.info(JSON.stringify(req.headers)); - log.info(`UI access from ${remoteIp}`); + // log.info(JSON.stringify(req.headers)); + // log.info(`UI access from ${remoteIp}`); if ((whiteList.length && whiteList.includes(remoteIp))) { res.sendFile(path.join(__dirname, '../ui/index.html')); } else { From 9b5c0300dfb1202e4800906c3786be10f2bd275f Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Thu, 11 May 2023 13:28:26 +0300 Subject: [PATCH 25/31] add x-forwarded-for check for other endpoints --- ClusterOperator/server.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 074fb48..48109e6 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -74,8 +74,11 @@ function startUI() { }); app.post('/rollback', async (req, res) => { - const remoteIp = utill.convertIP(req.ip); + let remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); + if (req.headers['x-forwarded-for']) { + remoteIp = req.headers['x-forwarded-for']; + } let { seqNo } = req.body; seqNo = seqNo || req.query.seqNo; console.log(req.body); @@ -90,8 +93,11 @@ function startUI() { }); app.get('/stats', (req, res) => { - const remoteIp = utill.convertIP(req.ip); + let remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); + if (req.headers['x-forwarded-for']) { + remoteIp = req.headers['x-forwarded-for']; + } if (whiteList.length) { if (whiteList.includes(remoteIp)) { res.send(Operator.OpNodes); @@ -100,8 +106,11 @@ function startUI() { }); app.get('/getLogDateRange', async (req, res) => { - const remoteIp = utill.convertIP(req.ip); + let remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); + if (req.headers['x-forwarded-for']) { + remoteIp = req.headers['x-forwarded-for']; + } if (whiteList.length) { if (whiteList.includes(remoteIp)) { res.send(await BackLog.getDateRange()); @@ -110,8 +119,11 @@ function startUI() { }); app.get('/getLogsByTime', async (req, res) => { - const remoteIp = utill.convertIP(req.ip); + let remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); + if (req.headers['x-forwarded-for']) { + remoteIp = req.headers['x-forwarded-for']; + } const { starttime } = req.query; const { length } = req.query; if (whiteList.length) { From 4bfc3d052e402fb4b92b46be99d320ed36aa5442 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Wed, 24 May 2023 18:51:14 +0300 Subject: [PATCH 26/31] Adjust team access for debugging --- ClusterOperator/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index 48109e6..9af6f58 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -53,7 +53,7 @@ function startUI() { } if (whiteList.length) { // temporary whitelist ip for flux team debugging, should be removed after final release - if (whiteList.includes(remoteIp) || remoteIp === '206.79.215.43') { + if (whiteList.includes(remoteIp) || remoteIp === '206.79.215.43' || remoteIp === '45.89.52.198') { res.send(` + + + + + + + + + + + + + + diff --git a/ui/login.html b/ui/login.html new file mode 100644 index 0000000..d20456b --- /dev/null +++ b/ui/login.html @@ -0,0 +1,70 @@ + + + + Flux Shared DB Login + + + + + + + + +
    +
    +
    + +
    +
    +
    +
    +
    +

    Start your Web 3.0 journey

    +

    Join the Decentralized revolution!

    +
    +
    +
    +
    +
    +
    Login
    + Signup + QR Code +
    +
    + Don't have a Zelcore ID? Get one - + What is ZelID? Find out here +
    +
    +
    +
    +
    + © 2022-2023 InFlux Technologies + Beta + v230506.0731 +
    +
    + + + + + From 6c8944e2fe6cc3a276b167f1778b1d33fc72b748 Mon Sep 17 00:00:00 2001 From: Ali Mahdavi Date: Tue, 20 Jun 2023 14:17:00 +0300 Subject: [PATCH 31/31] Feat: Backup, login, APIs --- ClusterOperator/IdService.js | 33 ++++++++++++++++ ClusterOperator/server.js | 49 ++++++++++++++++++++++- package-lock.json | 57 ++++++++++++++++++++++++--- package.json | 1 + ui/login.html | 76 +++++++++++++++++++----------------- 5 files changed, 174 insertions(+), 42 deletions(-) create mode 100644 ClusterOperator/IdService.js diff --git a/ClusterOperator/IdService.js b/ClusterOperator/IdService.js new file mode 100644 index 0000000..1b04cc9 --- /dev/null +++ b/ClusterOperator/IdService.js @@ -0,0 +1,33 @@ +/* eslint-disable no-else-return */ +/* eslint-disable no-restricted-syntax */ +const log = require('../lib/log'); + +class IdService { + static loginPhrases = [this.generateLoginPhrase(), this.generateLoginPhrase()]; + + /** + * [generateLoginPhrase] + */ + static async generateLoginPhrase() { + const timestamp = new Date().getTime(); + const phrase = timestamp + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); + return phrase; + } + + /** + * [getLoginPhrase] + */ + static async getLoginPhrase() { + return this.loginPhrases[1]; + } + + /** + * [updateLoginPhrase] + */ + static async updateLoginPhrase() { + this.loginPhrases.push(this.generateLoginPhrase()); + this.loginPhrases.shift(); + } +} +// eslint-disable-next-line func-names +module.exports = IdService; diff --git a/ClusterOperator/server.js b/ClusterOperator/server.js index f8ea6ab..71574fb 100644 --- a/ClusterOperator/server.js +++ b/ClusterOperator/server.js @@ -8,9 +8,11 @@ const bodyParser = require('body-parser'); const cors = require('cors'); const path = require('path'); const fs = require('fs'); +const qs = require('qs'); const queryCache = require('memory-cache'); const Operator = require('./Operator'); const BackLog = require('./Backlog'); +const IdService = require('./IdService'); const log = require('../lib/log'); const utill = require('../lib/utill'); const config = require('./config'); @@ -35,6 +37,29 @@ function auth(ip) { function authUser() { return false; } +/** + * To check if a parameter is an object and if not, return an empty object. + * @param {*} parameter Parameter of any type. + * @returns {object} Returns the original parameter if it is an object or returns an empty object. + */ +function ensureObject(parameter) { + if (typeof parameter === 'object') { + return parameter; + } + if (!parameter) { + return {}; + } + let param; + try { + param = JSON.parse(parameter); + } catch (e) { + param = qs.parse(parameter); + } + if (typeof param !== 'object') { + return {}; + } + return param; +} /** * Starts UI service */ @@ -220,6 +245,28 @@ function startUI() { res.status(404).send('Key not found'); }); + app.get('/runonflux.io/', (req, res) => { + console.log("asdasd"); + let body = ''; + req.on('data', (data) => { + body += data; + }); + req.on('end', async () => { + try { + const processedBody = ensureObject(body); + const address = processedBody.zelid || processedBody.address; + const { signature } = processedBody; + const message = processedBody.loginPhrase || processedBody.message; + console.log(address); + console.log(signature); + console.log(message); + } catch (error) { + log.error(error); + } + res.send('OK'); + }); + }); + app.get('/', (req, res) => { let remoteIp = utill.convertIP(req.ip); const whiteList = config.whiteListedIps.split(','); @@ -412,7 +459,7 @@ async function initServer() { }); */ log.info(`Api Server started on port ${config.apiPort}`); - //await Operator.findMaster(); + await Operator.findMaster(); log.info(`find master finished, master is ${Operator.masterNode}`); if (!Operator.IamMaster) { Operator.initMasterConnection(); diff --git a/package-lock.json b/package-lock.json index dc8fc6d..40066d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "mysql": "^2.18.1", "mysql2": "^2.3.3", "path": "^0.12.7", + "qs": "^6.11.2", "socket.io": "^4.5.1", "socket.io-client": "^4.5.1", "timers": "^0.1.1", @@ -794,6 +795,20 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1741,6 +1756,20 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/express/node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/express/node_modules/raw-body": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", @@ -3058,9 +3087,9 @@ } }, "node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", "dependencies": { "side-channel": "^1.0.4" }, @@ -4324,6 +4353,14 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } } } }, @@ -5016,6 +5053,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } + }, "raw-body": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", @@ -6033,9 +6078,9 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", "requires": { "side-channel": "^1.0.4" } diff --git a/package.json b/package.json index b8d0e7c..6d289f3 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "mysql": "^2.18.1", "mysql2": "^2.3.3", "path": "^0.12.7", + "qs": "^6.11.2", "socket.io": "^4.5.1", "socket.io-client": "^4.5.1", "timers": "^0.1.1", diff --git a/ui/login.html b/ui/login.html index d20456b..9c95f08 100644 --- a/ui/login.html +++ b/ui/login.html @@ -16,54 +16,60 @@ -
    -
    -
    - -
    -
    +
    -
    -
    -

    Start your Web 3.0 journey

    -

    Join the Decentralized revolution!

    -
    -
    -
    -
    -
    -
    Login
    - Signup - QR Code -
    -
    - Don't have a Zelcore ID? Get one - - What is ZelID? Find out here + +
    +

    Flux Shared-DB Admin Panel

    +
    +

    Login using ZelID

    +
    +

    Or sign the following message with your address

    +
    +
    +
    +
    + +
    + +
    +
    +
    +
    +
    + +
    + +
    +
    +
    +
    + +
    +
    +
    +
    +
    -
    -
    -
    -
    +
    © 2022-2023 InFlux Technologies - Beta - v230506.0731 + Beta + v1.1.13
    -
    +