From bffafaa82b43f33fe4bf122cf8a35cd77d892d20 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Tue, 8 Aug 2023 08:39:34 +0200 Subject: [PATCH 01/20] refactor: remove duplicate variable --- src/services/socket.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/socket.js b/src/services/socket.js index 81ccf07..cacb85a 100644 --- a/src/services/socket.js +++ b/src/services/socket.js @@ -21,7 +21,6 @@ class SocketService extends EventEmitter { */ this.serverSocket = null - this.clusterSocket = null this.clusteredCollectors = [] if (config.influxCollectors) { From a51deeea8d42972377e9316d682912e258db8496 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Tue, 8 Aug 2023 11:28:14 +0200 Subject: [PATCH 02/20] devops: execute prettier as pre-commit githook Add formatted files --- lefthook.yml | 12 ++++++++++++ src/exchange.js | 3 ++- src/models/index.js | 5 +++++ src/typedef.js | 8 ++++---- 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 lefthook.yml create mode 100644 src/models/index.js diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..6e48c30 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,12 @@ +# EXAMPLE USAGE +# Refer for explanation to following link: +# https://github.com/evilmartians/lefthook/blob/master/docs/full_guide.md +# + +pre-commit: + parallel: true + commands: + prettier: + glob: '*.{js,jsx,ts,tsx,css,scss,md,html,json,yml}' + run: npx prettier --write "{staged_files}" + stage_fixed: true diff --git a/src/exchange.js b/src/exchange.js index 3025411..fae599a 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -140,7 +140,8 @@ class Exchange extends EventEmitter { /** * Link exchange to a pair - * @param {*} pair + * @param {string} pair + * @param {boolean} returnConnectedEvent * @returns {Promise} */ async link(pair, returnConnectedEvent) { diff --git a/src/models/index.js b/src/models/index.js new file mode 100644 index 0000000..25c8e6c --- /dev/null +++ b/src/models/index.js @@ -0,0 +1,5 @@ +const Alert = require('./alert.model.js') + +module.exports = { + Alert +} diff --git a/src/typedef.js b/src/typedef.js index 2e69c63..4002506 100644 --- a/src/typedef.js +++ b/src/typedef.js @@ -49,10 +49,10 @@ * Keep track of a single connection (exchange + pair) * @typedef Connection * @type {{ - * exchange: string, - * pair: string, - * apiId: string, - * hit: number, + * exchange: string, + * pair: string, + * apiId: string, + * hit: number, * ping: number, * bar?: Bar * }} From 1d0376dc95f2f864c78397a9f3408f2daf8e8fbc Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Mon, 14 Aug 2023 19:43:08 +0200 Subject: [PATCH 03/20] doc(exchange): add returns parameter to unsub method --- src/exchange.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/exchange.js b/src/exchange.js index fae599a..349db07 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -933,10 +933,11 @@ class Exchange extends EventEmitter { } /** - * Unsub + * Unsubscribe * @param {WebSocket} api * @param {string} pair * @param {boolean} skipSending skip sending unsusbribe message + * @returns {boolean} */ async unsubscribe(api, pair, skipSending) { if (!this.markPairAsDisconnected(api, pair)) { From bb9722c42ff1260d2fdf0ed627c10ccf10b22f5b Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Tue, 15 Aug 2023 23:42:11 +0200 Subject: [PATCH 04/20] refactor(config): use optional chaining to determine configPath --- src/config.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/config.js b/src/config.js index c97ec28..14bc14c 100644 --- a/src/config.js +++ b/src/config.js @@ -232,15 +232,11 @@ if (process.argv.length > 2) { let userSettings = {} -const specificConfigFile = commandSettings.config - ? commandSettings.config - : commandSettings.configFile - ? commandSettings.configFile - : commandSettings.configPath - ? commandSettings.configPath - : null - -let configPath = specificConfigFile || 'config.json' +const configPath = + commandSettings?.config || + commandSettings?.configFile || + commandSettings?.configPath || + 'config.json' try { console.log('[init] using config file ' + configPath) From 2e22c2842ce14c640999d9cfe6033e7bb97e1bf1 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Fri, 15 Sep 2023 09:07:17 +0200 Subject: [PATCH 05/20] doc: add api schema --- docs/schema.yaml | 172 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 docs/schema.yaml diff --git a/docs/schema.yaml b/docs/schema.yaml new file mode 100644 index 0000000..9e76f64 --- /dev/null +++ b/docs/schema.yaml @@ -0,0 +1,172 @@ +openapi: 3.0.3 +info: + title: Aggr-server API + version: 1.0.0 +servers: + - url: https://api.aggr.trade/ + description: AGGR Trade server + - url: http://localhost:3000/ + description: Default local server url +paths: + /alert: + post: + summary: Create or update an alert + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AlertPayload' + responses: + '201': + description: Alert created or updated successfully + content: + application/json: + schema: + $ref: '#/components/schemas/AlertResponse' + '400': + description: Invalid alert payload + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /products: + get: + summary: Get the list of products + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ProductsResponse' + + /historical/{from}/{to}/{timeframe}/{markets}: + get: + summary: Get historical data + parameters: + - in: path + name: from + schema: + type: integer + required: true + description: Start timestamp of the historical data + - in: path + name: to + schema: + type: integer + required: true + description: End timestamp of the historical data + - in: path + name: timeframe + schema: + type: string + required: true + description: Timeframe of the historical data + - in: path + name: markets + schema: + type: string + description: Markets to fetch historical data for + required: true + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/HistoricalDataResponse' + '400': + description: Invalid request or missing interval + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: No results found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' +components: + schemas: + AlertPayload: + type: object + properties: + endpoint: + type: string + keys: + type: array + items: + type: string + market: + type: string + price: + type: number + required: + - endpoint + - keys + - market + - price + AlertResponse: + type: object + + Error: + type: object + properties: + error: + type: string + ProductsResponse: + type: array + items: + type: string + HistoricalDataResponse: + type: object + properties: + format: + type: string + columns: + type: object + properties: + time: + type: integer + cbuy: + type: integer + close: + type: integer + csell: + type: integer + high: + type: integer + lbuy: + type: integer + low: + type: integer + lsell: + type: integer + market: + type: string + open: + type: integer + vbuy: + type: number + vsell: + type: number + nullable: true + results: + type: array + items: + type: array + items: + oneOf: + - type: integer + - type: number + - type: string + nullable: true + \ No newline at end of file From 832b79a5d6f3efc175fa2e8dea7328532afe1d90 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Fri, 15 Sep 2023 12:41:56 +0200 Subject: [PATCH 06/20] fix: config.json couldn't be parsed --- src/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.js b/src/config.js index 14bc14c..33d2b0a 100644 --- a/src/config.js +++ b/src/config.js @@ -232,7 +232,7 @@ if (process.argv.length > 2) { let userSettings = {} -const configPath = +let configPath = commandSettings?.config || commandSettings?.configFile || commandSettings?.configPath || From 3316b455f2dda4f2b63673b5382f2e16219d14d3 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Sun, 17 Sep 2023 00:36:31 +0200 Subject: [PATCH 07/20] doc: refactor Trade typedef --- src/typedef.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/typedef.js b/src/typedef.js index 4002506..a6a7667 100644 --- a/src/typedef.js +++ b/src/typedef.js @@ -1,7 +1,13 @@ /** - * A trade - * @typedef Trade - * @type {{exchange: string, pair: string, timestamp: number, price: number, size: number, side: number, liquidation: boolean?}} + * Represents a trade. + * @typedef {Object} Trade + * @property {string} exchange - The exchange where the trade occurred. + * @property {string} pair - The trading pair involved in the trade. + * @property {number} timestamp - The timestamp of the trade. + * @property {number} price - The price at which the trade occurred. + * @property {number} size - The size or quantity of the trade. + * @property {number} side - The side of the trade (1 for buy, 2 for sell, for example). + * @property {boolean} [liquidation] - Optional. Indicates whether the trade was a liquidation. */ /** From a54b3604ce9f44c31a814238a6e3830456dc79ea Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Sun, 17 Sep 2023 00:47:09 +0200 Subject: [PATCH 08/20] refacto: dispatchRawTrades Removed exchange id argument as same info could be extracted from the trade object --- src/server.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/server.js b/src/server.js index 6fc9f85..25335d7 100644 --- a/src/server.js +++ b/src/server.js @@ -249,13 +249,10 @@ class Server extends EventEmitter { this.dispatchAggregateTrade.bind(this, exchange.id) ) } else { - exchange.on('trades', this.dispatchRawTrades.bind(this, exchange.id)) + exchange.on('trades', this.dispatchRawTrades.bind(this)) } - exchange.on( - 'liquidations', - this.dispatchRawTrades.bind(this, exchange.id) - ) + exchange.on('liquidations', this.dispatchRawTrades.bind(this)) exchange.on('disconnected', (pair, apiId, apiLength) => { const id = exchange.id + ':' + pair @@ -991,12 +988,16 @@ class Server extends EventEmitter { }) } - dispatchRawTrades(exchange, data) { - for (let i = 0; i < data.length; i++) { - const trade = data[i] + /** + * @param {Trade[]} trades + */ + + dispatchRawTrades(trades) { + for (let i = 0; i < trades.length; i++) { + const trade = trades[i] if (!trade.liquidation) { - const identifier = exchange + ':' + trade.pair + const identifier = trade.exchange + ':' + trade.pair // ping connection connections[identifier].hit++ @@ -1011,9 +1012,9 @@ class Server extends EventEmitter { if (config.broadcast) { if (!config.broadcastAggr && !config.broadcastDebounce) { - this.broadcastTrades(data) + this.broadcastTrades(trades) } else { - Array.prototype.push.apply(this.delayedForBroadcast, data) + Array.prototype.push.apply(this.delayedForBroadcast, trades) } } } From d803e13363c3cc1729e0beb05966e9a840382fac Mon Sep 17 00:00:00 2001 From: Tucsky Date: Wed, 5 Apr 2023 00:54:19 +0200 Subject: [PATCH 09/20] fix offline alerts --- index.js | 9 ++++++++ src/config.js | 15 ++++++++++++- src/helper.js | 23 ++++++++++---------- src/server.js | 15 +++++++++++++ src/services/alert.js | 43 ++++++++++++++++++++++++++----------- src/services/catalog.js | 40 +++++++++++++++++----------------- src/services/connections.js | 27 ++++++++++++++++------- src/storage/influx.js | 16 ++++++++------ 8 files changed, 128 insertions(+), 60 deletions(-) diff --git a/index.js b/index.js index 0b1c6dd..9bd34b6 100644 --- a/index.js +++ b/index.js @@ -104,4 +104,13 @@ if (process.env.pmx) { reply(`FAILED to disconnect ${markets} (${err.message})`) }) }) + tx2.action('triggeralert', function (user, reply) { + // offline webpush testing + try { + const result = server.triggerAlert(user) + reply(result) + } catch (error) { + console.error('FAILED to trigger alert', user, error.message) + } + }) } diff --git a/src/config.js b/src/config.js index 33d2b0a..6fe8629 100644 --- a/src/config.js +++ b/src/config.js @@ -197,7 +197,8 @@ const defaultConfig = { privateVapidKey: null, alertExpiresAfter: 1000 * 60 * 60 * 24 * 7, alertEndpointExpiresAfter: 1000 * 60 * 60 * 24 * 30, - priceIndexesBlacklist: [], + indexExchangeBlacklist: [], + indexQuoteWhitelist: ['USD','USDT','BUSD','USDC'], // verbose debug: false @@ -313,6 +314,18 @@ if (config.whitelist && config.whitelist === 'string') { config.whitelist = [] } +if (typeof config.indexExchangeBlacklist === 'string' && config.indexExchangeBlacklist.trim().length) { + config.indexExchangeBlacklist = config.indexExchangeBlacklist.split(',').map(a => a.trim()) +} else if (!Array.isArray(config.indexExchangeBlacklist)) { + config.indexExchangeBlacklist = [] +} + +if (typeof config.indexQuoteWhitelist === 'string' && config.indexQuoteWhitelist.trim().length) { + config.indexQuoteWhitelist = config.indexQuoteWhitelist.split(',').map(a => a.trim()) +} else if (!Array.isArray(config.indexQuoteWhitelist)) { + config.indexQuoteWhitelist = [] +} + if (config.pair) { config.pairs = Array.isArray(config.pair) ? config.pair diff --git a/src/helper.js b/src/helper.js index b575b56..6608d62 100644 --- a/src/helper.js +++ b/src/helper.js @@ -128,21 +128,20 @@ module.exports = { formatAmount(amount, decimals) { const negative = amount < 0 - - if (negative) { - amount = Math.abs(amount) - } - - if (amount >= 1000000) { - amount = +(amount / 1000000).toFixed(isNaN(decimals) ? 1 : decimals) + 'M' - } else if (amount >= 100000) { - amount = +(amount / 1000).toFixed(isNaN(decimals) ? 0 : decimals) + 'K' + + amount = Math.abs(amount) + + if (amount >= 1000000000) { + amount = + +(amount / 1000000000).toFixed(isNaN(decimals) ? 1 : decimals) + ' B' + } else if (amount >= 1000000) { + amount = +(amount / 1000000).toFixed(isNaN(decimals) ? 1 : decimals) + ' M' } else if (amount >= 1000) { - amount = +(amount / 1000).toFixed(isNaN(decimals) ? 1 : decimals) + 'K' + amount = +(amount / 1000).toFixed(isNaN(decimals) ? 1 : decimals) + ' K' } else { - amount = +amount.toFixed(4) + amount = +amount.toFixed(isNaN(decimals) ? 2 : decimals) } - + if (negative) { return '-' + amount } else { diff --git a/src/server.js b/src/server.js index 25335d7..f36d7eb 100644 --- a/src/server.js +++ b/src/server.js @@ -1139,6 +1139,21 @@ class Server extends EventEmitter { return output } + + triggerAlert(user) { + for (const market in alertService.alerts) { + for (const range in alertService.alerts[market]) { + for (const alert of alertService.alerts[market][range]) { + if (alertService.alertEndpoints[alert.endpoint] && alertService.alertEndpoints[alert.endpoint].user === user) { + alertService.sendAlert(alert, alert.market, Date.now()) + return `${alert.market} @${alert.price}` + } + } + } + } + + throw new Error(`alert not found for user ${user}`) + } } module.exports = Server diff --git a/src/services/alert.js b/src/services/alert.js index d041525..5ab68c3 100644 --- a/src/services/alert.js +++ b/src/services/alert.js @@ -2,7 +2,7 @@ const config = require('../config') const socketService = require('./socket') const persistenceService = require('./persistence') const EventEmitter = require('events') -const { indexes, getIndex } = require('./connections') +const { indexes, getIndex, debugIndexes } = require('./connections') const { getHms, sleep, ago } = require('../helper') const webPush = require('web-push') @@ -174,7 +174,7 @@ class AlertService extends EventEmitter { this.alertEndpoints[alert.endpoint].timestamp = now - console.log(`[alert/${alert.user}] create alert ${market} @${alert.price}`) + console.log(`[alert/${alert.user}] create alert ${market} @${alert.price} (ajusted to ${alert.price - (priceOffset || 0)})`) if (alert.message) { console.log(`\t 💬 ${alert.message}`) @@ -262,7 +262,9 @@ class AlertService extends EventEmitter { this.alerts[activeAlert.market][rangePrice].splice(index, 1) console.log( - `[alert/${activeAlert.user}] move alert on ${activeAlert.market} @${activeAlert.price} -> ${newAlert.price}` + `[alert/${activeAlert.user}] move alert on ${activeAlert.market} @${activeAlert.price} -> ${newAlert.price} (ajusted to ${ + activeAlert.price - (priceOffset || 0) + })` ) const now = Date.now() @@ -502,7 +504,7 @@ class AlertService extends EventEmitter { } } - sendAlert(alert, market, elapsedTime, direction) { + sendAlert(alert, market, timestamp, direction) { if (!this.alertEndpoints[alert.endpoint]) { console.error( `[alert/send] attempted to send alert without matching endpoint`, @@ -511,6 +513,8 @@ class AlertService extends EventEmitter { return } + const elapsedTime = timestamp - alert.timestamp + console.log( `[alert/send/${ this.alertEndpoints[alert.endpoint].user @@ -520,11 +524,26 @@ class AlertService extends EventEmitter { alert.triggered = true let message + let title if (alert.message) { + if (direction > 0) { + title = `${market} 📈` + } else if (direction < 0) { + title = `${market} 📉` + } else { + title = market + } message = alert.message } else { - message = `Price crossed ${alert.price}` + title = market + if (direction > 0) { + message = `📈 ${alert.price}` + } else if (direction < 0) { + message = `📉 ${alert.price}` + } else { + message = `Price crossed ${alert.price}` + } } const payload = JSON.stringify({ @@ -554,7 +573,7 @@ class AlertService extends EventEmitter { contentEncoding: 'aes128gcm' }) .then(() => { - return sleep(1000) + return sleep(100) }) .catch(err => { console.error( @@ -607,14 +626,14 @@ class AlertService extends EventEmitter { continue } - const isTriggered = - alert.priceCompare <= high && alert.priceCompare >= low + const isTriggered = alert.priceCompare >= low && alert.priceCompare <= high if (isTriggered) { - console.log( - `[alert/checkPriceCrossover] ${alert.price} (ajusted to ${alert.priceCompare}) ${market} ${low} <-> ${high}` - ) - await this.sendAlert(alert, market, now - alert.timestamp, direction) + console.log(`[alert/checkPriceCrossover] ${alert.price} (ajusted to ${alert.priceCompare}) ${market} ${low} <-> ${high}`) + if (debugIndexes[market]) { + console.log(`debug index ${market}: ${debugIndexes[market].join(',')}`) + } + await this.sendAlert(alert, market, now, direction) if (this.unregisterAlert(alert, market, true)) { i-- diff --git a/src/services/catalog.js b/src/services/catalog.js index 1416635..a4ca136 100644 --- a/src/services/catalog.js +++ b/src/services/catalog.js @@ -2,10 +2,11 @@ const axios = require('axios') const fs = require('fs') const { ensureDirectoryExists } = require('../helper') -const baseQuoteLookupKnown = new RegExp( - `^([A-Z0-9]{3,})[-/:_]?(USDT|USDC|TUSD|BUSD|USDD|USDK|USDP)$|^([A-Z0-9]{2,})[-/:]?(UST|EUR|USD)$` -) -const baseQuoteLookupOthers = new RegExp(`^([A-Z0-9]{2,})[-/_]?([A-Z0-9]{3,})$`) +const stablecoins = ['USDT', 'USDC', 'TUSD', 'BUSD', 'USDD', 'USDK', 'USDP', 'UST'] +const currencies = ['EUR', 'USD', 'GBP', 'AUD', 'CAD', 'CHF'] +const currencyPairLookup = new RegExp(`^([A-Z0-9]{2,})[-/:]?(${currencies.join('|')})$`) +const stablecoinPairLookup = new RegExp(`^([A-Z0-9]{2,})[-/:_]?(${stablecoins.join('|')})$`) +const simplePairLookup = new RegExp(`^([A-Z0-9]{2,})[-/_]?([A-Z0-9]{3,})$`) require('../typedef') @@ -184,10 +185,7 @@ module.exports.parseMarket = function (market, noStable = true) { } else if (exchangeId === 'DERIBIT') { localSymbol = localSymbol.replace(/_(\w+)-PERPETUAL/i, '$1') } else if (exchangeId === 'BITGET') { - localSymbol = localSymbol - .replace('USD_DMCBL', 'USD') - .replace('PERP_CMCBL', 'USDC') - .replace(/_.*/, '') + localSymbol = localSymbol.replace('USD_DMCBL', 'USD').replace('PERP_CMCBL', 'USDC').replace(/_.*/, '') } else if (exchangeId === 'KUCOIN') { localSymbol = localSymbol.replace(/M$/, '') } @@ -202,21 +200,23 @@ module.exports.parseMarket = function (market, noStable = true) { let localSymbolAlpha = localSymbol.replace(/[-_/:]/, '') let match - - match = localSymbol.match(baseQuoteLookupKnown) + if (exchangeId !== 'BINANCE') { + match = localSymbol.match(currencyPairLookup) + } if (!match) { - match = localSymbolAlpha.match(baseQuoteLookupOthers) - } + match = localSymbol.match(stablecoinPairLookup) - if ( - !match && - (exchangeId === 'DERIBIT' || exchangeId === 'FTX' || exchangeId === 'HUOBI') - ) { - match = localSymbolAlpha.match(/(\w+)[^a-z0-9]/i) + if (!match) { + match = localSymbolAlpha.match(simplePairLookup) + } - if (match) { - match[2] = match[1] + if (!match && (exchangeId === 'DERIBIT' || exchangeId === 'HUOBI')) { + match = localSymbolAlpha.match(/(\w+)[^a-z0-9]/i) + + if (match) { + match[2] = match[1] + } } } @@ -246,6 +246,6 @@ module.exports.parseMarket = function (market, noStable = true) { pair: symbol, local: localSymbolAlpha, exchange: exchangeId, - type + type, } } diff --git a/src/services/connections.js b/src/services/connections.js index b2f21db..a29e10f 100644 --- a/src/services/connections.js +++ b/src/services/connections.js @@ -11,6 +11,7 @@ let saveConnectionsTimeout = null * @type {{[id: string]: Connection}} */ const connections = (module.exports.connections = {}) +const debugIndexes = (module.exports.debugIndexes = {}) /** * @type {{[id: string]: Boolean}} @@ -30,7 +31,7 @@ const indexes = (module.exports.indexes = []) for (const market of config.pairs) { const product = parseMarket(market) - if (config.priceIndexesBlacklist.indexOf(product.exchange) !== -1) { + if (config.indexExchangeBlacklist.indexOf(product.exchange) !== -1 || config.indexQuoteWhitelist.indexOf(product.quote) === -1) { continue } @@ -408,6 +409,8 @@ module.exports.updateIndexes = async function (ranges, callback) { let close = 0 let nbSources = 0 + debugIndexes[index.id] = [] + for (const market of index.markets) { if ( !connections[market] || @@ -420,11 +423,13 @@ module.exports.updateIndexes = async function (ranges, callback) { if (ranges[market]) { high += ranges[market].high low += ranges[market].low + debugIndexes[index.id].push(`${market}:rg:${ranges[market].low}-${ranges[market].high}`) connections[market].low = ranges[market].low connections[market].high = ranges[market].high } else { high += connections[market].close low += connections[market].close + debugIndexes[index.id].push(`${market}:co:${connections[market].close}-${connections[market].close}`) connections[market].low = connections[market].close connections[market].high = connections[market].close } @@ -437,14 +442,20 @@ module.exports.updateIndexes = async function (ranges, callback) { continue } - index.price = close / nbSources + const direction = close / nbSources > index.price ? 1 : -1 - await callback( - index.id, - high / nbSources, - low / nbSources, - index.price > open ? 1 : -1 - ) + if (direction > 0) { + high = high /= nbSources + low = Math.min(index.high || Infinity, low / nbSources) + } else { + high = Math.max(index.low || -Infinity, high / nbSources) + low = low / nbSources + } + + index.price = close / nbSources + index.high = high + index.low = low + await callback(index.id, index.high, index.low, direction) } } diff --git a/src/storage/influx.js b/src/storage/influx.js index 13ebe88..80378be 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -389,14 +389,16 @@ class InfluxStorage { Math.floor(trade.timestamp / config.influxTimeframe) * config.influxTimeframe - if (!ranges[market]) { - ranges[market] = { - low: trade.price, - high: trade.price + if (!trade.liquidation) { + if (!ranges[market]) { + ranges[market] = { + low: trade.price, + high: trade.price, + } + } else { + ranges[market].low = Math.min(ranges[market].low, trade.price) + ranges[market].high = Math.max(ranges[market].high, trade.price) } - } else { - ranges[market].low = Math.min(ranges[market].low, trade.price) - ranges[market].high = Math.max(ranges[market].high, trade.price) } if (!bars[market] || bars[market].time !== tradeFlooredTime) { From d14721d3c7d71405e4a6ad18c1dd936fe29f6059 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Sat, 15 Apr 2023 02:48:01 +0200 Subject: [PATCH 10/20] use data-stream binance endpoint --- src/exchanges/binance.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index e02129c..0fc09fb 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -14,7 +14,7 @@ class Binance extends Exchange { PRODUCTS: 'https://data.binance.com/api/v3/exchangeInfo' } - this.url = () => `wss://stream.binance.com:9443/ws` + this.url = () => `wss://data-stream.binance.com:9443/ws` } formatProducts(data) { From e79c5edeba1aaa6bd6c8146bf2b4f6244b147685 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Sat, 7 Oct 2023 16:14:38 +0200 Subject: [PATCH 11/20] chore: add npm clean script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 674879c..21dde82 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "dev": "pm2-dev ecosystem.config.dev.yaml", "generateVapidKeys": "node scripts/web-push/generateVapidKeys", "lint:fix": "npx prettier index.js ./src/* --write", + "clean": "rm -rf node-modules && npm clean cache --force", "docker:server:build": "docker-compose build", "docker:server:up": "docker-compose up -d" }, From dbf57b8244cc8c9b9890959bfd31ac5fd3403650 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Tue, 10 Oct 2023 15:21:23 +0200 Subject: [PATCH 12/20] fix mexc spot listing fix bitfinex swapping pair id recover okex liquidations --- scripts/coinalize/utils.js | 28 ++-- src/config.js | 6 +- src/exchanges/binance.js | 8 +- src/exchanges/bitfinex.js | 46 +++---- src/exchanges/huobi.js | 4 +- src/exchanges/mexc.js | 28 ++-- src/exchanges/okex.js | 260 +++++++++++++++++-------------------- src/services/catalog.js | 2 +- src/storage/influx.js | 24 +++- 9 files changed, 197 insertions(+), 209 deletions(-) diff --git a/scripts/coinalize/utils.js b/scripts/coinalize/utils.js index ba24de6..0b8ab90 100644 --- a/scripts/coinalize/utils.js +++ b/scripts/coinalize/utils.js @@ -29,19 +29,21 @@ const COINALIZE_REQ_KEY_PATH = 'products/coinalize.key' const BARS_PER_REQUEST = 300 const baseHeaders = { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9,fr-FR;q=0.8,fr;q=0.7', - 'cache-control': 'no-cache', - pragma: 'no-cache', - 'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - cookie: 'p_sid=s%3ARY_H7HfwdvenT8Pz2XjPjBq-mHVLMsmL.ONSYPlNqShlYwQoVZ4Ez8igcJZ1r1wAG6pw7HQ%2FJ2WE', - Referer: 'https://coinalyze.net/bitcoin/usd/binance/btcusd_perp/price-chart-live/', - 'Referrer-Policy': 'strict-origin-when-cross-origin', + "accept": "*/*", + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "content-type": "application/json", + "pragma": "no-cache", + "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + "x-requested-with": "XMLHttpRequest", + "cookie": "cookies_accepted=1; p_sid=s%3A944WOtc4HxnJXvGuR5KtvuG1yIpHEYeT.r3vU6rPl3k4tjB4xGW472GoQPOjJi%2FbfMIyW0R1bRkM", + "Referer": "https://coinalyze.net/bitcoin/usd/binance/btcusd_perp/price-chart-live/", + "Referrer-Policy": "strict-origin-when-cross-origin" } const baseJSONHeaders = { diff --git a/src/config.js b/src/config.js index 6fe8629..6b5d49d 100644 --- a/src/config.js +++ b/src/config.js @@ -234,9 +234,9 @@ if (process.argv.length > 2) { let userSettings = {} let configPath = - commandSettings?.config || - commandSettings?.configFile || - commandSettings?.configPath || + commandSettings.config || + commandSettings.configFile || + commandSettings.configPath || 'config.json' try { diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index 0fc09fb..83d71d4 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -11,10 +11,10 @@ class Binance extends Exchange { this.subscriptions = {} this.endpoints = { - PRODUCTS: 'https://data.binance.com/api/v3/exchangeInfo' + PRODUCTS: 'https://data-api.binance.vision/api/v3/exchangeInfo', } - this.url = () => `wss://data-stream.binance.com:9443/ws` + this.url = () => `wss://data-stream.binance.vision:9443/ws` } formatProducts(data) { @@ -98,10 +98,12 @@ class Binance extends Exchange { const startTime = range.from const below1HEndTime = Math.min(range.to, startTime + 1000 * 60 * 60) - const endpoint = `https://data.binance.com/api/v3/aggTrades?symbol=${range.pair.toUpperCase()}&startTime=${ + const endpoint = `https://data-api.binance.vision/api/v3/aggTrades?symbol=${range.pair.toUpperCase()}&startTime=${ startTime + 1 }&endTime=${below1HEndTime}&limit=1000` + console.log(endpoint) + return axios .get(endpoint) .then(response => { diff --git a/src/exchanges/bitfinex.js b/src/exchanges/bitfinex.js index 6838382..44d31ab 100644 --- a/src/exchanges/bitfinex.js +++ b/src/exchanges/bitfinex.js @@ -63,9 +63,7 @@ class Bitfinex extends Exchange { } if (api._connected.length === 0) { - const chanId = Object.keys(this.channels).find( - id => this.channels[id].name === 'status' - ) + const chanId = Object.keys(this.channels[api.id]).find((chanId) => this.channels[api.id][chanId].name === 'status') if (chanId) { api.send( @@ -75,13 +73,12 @@ class Bitfinex extends Exchange { }) ) - delete this.channels[chanId] + delete this.channels[api.id][chanId] } } - const channelsToUnsubscribe = Object.keys(this.channels).filter( - id => - this.channels[id].pair === pair && this.channels[id].apiId === api.id + const channelsToUnsubscribe = Object.keys(this.channels[api.id]).filter( + (chanId) => this.channels[api.id][chanId].pair === pair ) if (!channelsToUnsubscribe.length) { @@ -92,14 +89,14 @@ class Bitfinex extends Exchange { return } - for (let id of channelsToUnsubscribe) { + for (let chanId of channelsToUnsubscribe) { api.send( JSON.stringify({ event: 'unsubscribe', - chanId: id + chanId, }) ) - delete this.channels[id] + delete this.channels[api.id][chanId] } } @@ -107,20 +104,21 @@ class Bitfinex extends Exchange { const json = JSON.parse(event.data) if (json.event === 'subscribed' && json.chanId) { - // console.debug(`[${this.id}] register channel ${json.chanId} (${json.channel}:${json.pair})`) - this.channels[json.chanId] = { + // console.debug(`[${this.id}.${api.id}] register channel ${json.chanId} (${json.channel}:${json.pair})`) + this.channels[api.id][json.chanId] = { name: json.channel, - pair: json.pair, - apiId: api.id + pair: json.pair } return } - if (!this.channels[json[0]] || json[1] === 'hb') { + const chanId = json[0] + + if (!this.channels[api.id][chanId] || json[1] === 'hb') { return } - const channel = this.channels[json[0]] + const channel = this.channels[api.id][chanId] if (!channel.hasSentInitialMessage) { // console.debug(`[${this.id}] skip first payload ${channel.name}:${channel.pair}`) @@ -240,18 +238,12 @@ class Bitfinex extends Exchange { }) } - onApiRemoved(api) { - console.log( - `[${this.id}] has ${Object.keys(this.channels).length} channels` - ) - - const apiChannels = Object.keys(this.channels).filter( - id => this.channels[id].apiId === api.id - ) + onApiCreated(api) { + this.channels[api.id] = {} + } - for (const id of apiChannels) { - delete this.channels[id] - } + onApiRemoved(api) { + delete this.channels[api.id] } } diff --git a/src/exchanges/huobi.js b/src/exchanges/huobi.js index aeedeee..15e3ded 100644 --- a/src/exchanges/huobi.js +++ b/src/exchanges/huobi.js @@ -183,8 +183,8 @@ class Huobi extends Exchange { return { exchange: this.id, pair: pair, - timestamp: +new Date(), - price: this.prices[pair] || trade.price, + timestamp: trade.created_at, + price: +trade.price || this.prices[pair], size: +trade.amount, side: trade.direction, liquidation: true diff --git a/src/exchanges/mexc.js b/src/exchanges/mexc.js index a2c6abd..e88a4ac 100644 --- a/src/exchanges/mexc.js +++ b/src/exchanges/mexc.js @@ -11,9 +11,8 @@ class Mexc extends Exchange { this.endpoints = { PRODUCTS: [ - 'https://api.mexc.com/api/v3/defaultSymbols', - 'https://contract.mexc.com/api/v1/contract/detail' - ] + 'https://api.mexc.com/api/v3/exchangeInfo', + 'https://contract.mexc.com/api/v1/contract/detail'] } this.url = pair => { @@ -29,18 +28,19 @@ class Mexc extends Exchange { const products = [] const contractSizes = {} const inversed = {} + const [spot, perp] = responses - for (const response of responses) { - const type = ['spot', 'contract'][responses.indexOf(response)] - - for (const product of response.data) { - if (type === 'contract') { - products.push(product.symbol) - contractSizes[product.symbol] = product.contractSize - inversed[product.symbol] = product.quoteCoin === product.settleCoin - } else { - products.push(product) - } + if (spot) { + for (const product of spot.symbols) { + products.push(product.symbol) + } + } + + if (perp) { + for (const product of perp.data) { + products.push(product.symbol) + contractSizes[product.symbol] = product.contractSize + inversed[product.symbol] = product.quoteCoin === product.settleCoin } } diff --git a/src/exchanges/okex.js b/src/exchanges/okex.js index a515cfc..f66c048 100644 --- a/src/exchanges/okex.js +++ b/src/exchanges/okex.js @@ -10,6 +10,7 @@ class Okex extends Exchange { this.id = 'OKEX' this.endpoints = { + LIQUIDATIONS: 'https://www.okx.com/api/v5/public/liquidation-orders', PRODUCTS: [ 'https://www.okex.com/api/v5/public/instruments?instType=SPOT', 'https://www.okex.com/api/v5/public/instruments?instType=FUTURES', @@ -35,11 +36,11 @@ class Okex extends Exchange { const type = product.instType const pair = product.instId + if (type === 'FUTURES') { // futures specs[pair] = +product.ctVal - types[pair] = 'futures' aliases[pair] = product.alias if (product.ctType === 'inverse') { @@ -49,15 +50,13 @@ class Okex extends Exchange { // swap specs[pair] = +product.ctVal - types[pair] = 'swap' if (product.ctType === 'inverse') { inversed[pair] = true } - } else { - types[pair] = 'spot' } + types[pair] = type products.push(pair) } } @@ -81,19 +80,6 @@ class Okex extends Exchange { return } - const type = this.types[pair] - - if (type !== 'spot') { - this.liquidationProducts.push(pair) - this.liquidationProductsReferences[pair] = Date.now() - - if (this.liquidationProducts.length === 1) { - this.startLiquidationTimer() - } - - console.debug(`[${this.id}] listen ${pair} for liquidations`) - } - api.send( JSON.stringify({ op: 'subscribe', @@ -105,6 +91,20 @@ class Okex extends Exchange { ] }) ) + + if (this.types[pair] !== 'SPOT') { + api.send( + JSON.stringify({ + op: 'subscribe', + args: [ + { + channel: 'liquidation-orders', + instType: this.types[pair] + } + ] + }) + ) + } } /** @@ -117,18 +117,6 @@ class Okex extends Exchange { return } - const type = this.types[pair] - - if (type !== 'spot' && this.liquidationProducts.indexOf(pair) !== -1) { - this.liquidationProducts.splice(this.liquidationProducts.indexOf(pair), 1) - - console.debug(`[${this.id}] unsubscribe ${pair} from liquidations loop`) - - if (this.liquidationProducts.length === 0) { - this.stopLiquidationTimer() - } - } - api.send( JSON.stringify({ op: 'unsubscribe', @@ -140,6 +128,20 @@ class Okex extends Exchange { ] }) ) + + if (this.types[pair] !== 'SPOT') { + api.send( + JSON.stringify({ + op: 'subscribe', + args: [ + { + channel: 'liquidation-orders', + instType: this.types[pair] + } + ] + }) + ) + } } onMessage(event, api) { @@ -149,6 +151,25 @@ class Okex extends Exchange { return } + if (json.arg.channel === 'liquidation-orders') { + const liqs = json.data.reduce((acc, pairData) => { + if (api._connected.indexOf(pairData.instId) === -1) { + return acc + } + + return acc.concat( + pairData.details.map(liquidation => + this.formatLiquidation(liquidation, pairData.instId) + ) + ) + }, []) + + return this.emitLiquidations( + api.id, + liqs + ) + } + return this.emitTrades( api.id, json.data.map(trade => this.formatTrade(trade)) @@ -176,55 +197,90 @@ class Okex extends Exchange { } } - formatLiquidation(trade, pair) { + formatLiquidation(liquidation, pair) { const size = - (trade.sz * this.specs[pair]) / (this.inversed[pair] ? trade.bkPx : 1) - //console.debug(`[${this.id}] okex liquidation at ${new Date(+trade.ts).toISOString()}`) + (liquidation.sz * this.specs[pair]) / + (this.inversed[pair] ? liquidation.bkPx : 1) return { exchange: this.id, pair: pair, - timestamp: +trade.ts, - price: +trade.bkPx, + timestamp: +liquidation.ts, + price: +liquidation.bkPx, size: size, - side: trade.side, + side: liquidation.side, liquidation: true } } - startLiquidationTimer() { - if (this._liquidationInterval) { - return - } + getLiquidationsUrl(range) { + // after query param = before + // (get the 100 trades prededing endTimestamp) + return `${this.endpoints.LIQUIDATIONS}?instId=${range.pair}&instType=SWAP&uly=${range.pair.replace('-SWAP', '')}&state=filled&after=${range.to}` + } - console.debug(`[${this.id}] start liquidation timer`) + /** + * Fetch pair liquidations before timestamp + * @param {*} range + * @returns + */ + async fetchLiquidationOrders(range) { + const url = this.getLiquidationsUrl(range) + console.log(url) + try { + const response = await axios.get(url) + if (response.data.data && response.data.data.length) { + return response.data.data[0].details + } + return [] + } catch (error) { + throw new Error(`Error fetching data: ${error}`) + } + } - this._liquidationProductIndex = 0 + async fetchAllLiquidationOrders(range) { + const allLiquidations = [] - this._liquidationInterval = setInterval( - this.fetchLatestLiquidations.bind(this), - 2500 - ) - } + while (true) { + console.log(`[${this.id}] fetch all ${range.pair} liquidations in`, new Date(range.from).toISOString(), new Date(range.to).toISOString()) + const liquidations = await this.fetchLiquidationOrders(range) - stopLiquidationTimer() { - if (!this._liquidationInterval) { - return - } + if (!liquidations || liquidations.length === 0) { + console.log('received', liquidations.length, 'liquidations -> break') + return allLiquidations + } + console.log('received', liquidations.length, 'liquidations -> process') - console.debug(`[${this.id}] stop liquidation timer`) + console.log('first liq @ ', new Date(+liquidations[0].ts).toISOString(), new Date(+liquidations[liquidations.length - 1].ts).toISOString()) + for (const liquidation of liquidations) { + if (liquidation.ts < range.from) { + console.log(`liquidation ${liquidations.indexOf(liquidation) + 1}/${liquidations.length} is outside range -> break`) + return allLiquidations + } - clearInterval(this._liquidationInterval) + allLiquidations.push(liquidation) + } - delete this._liquidationInterval + // new range + console.log(`[${this.id}] set new end date to last liquidation`, new Date(+liquidations[liquidations.length - 1].ts).toISOString()) + range.to = +liquidations[liquidations.length - 1].ts + } } async getMissingTrades(range, totalRecovered = 0, beforeTradeId) { + if (this.types[range.pair] !== 'SPOT' && !beforeTradeId) { + const liquidations = await this.fetchAllLiquidationOrders({ ...range }) + console.log(`[${this.id}.recoverMissingTrades] +${liquidations.length} liquidations for ${range.pair}`) + + if (liquidations.length) { + this.emitLiquidations(null, liquidations) + } + } + let endpoint if (beforeTradeId) { - endpoint = `https://www.okex.com/api/v5/market/history-trades?instId=${ - range.pair - }&limit=500${beforeTradeId ? '&after=' + beforeTradeId : ''}` + endpoint = `https://www.okex.com/api/v5/market/history-trades?instId=${range.pair + }&limit=500${beforeTradeId ? '&after=' + beforeTradeId : ''}` } else { endpoint = `https://www.okex.com/api/v5/market/trades?instId=${range.pair}&limit=500` } @@ -237,11 +293,12 @@ class Okex extends Exchange { response.data.data[response.data.data.length - 1].tradeId const earliestTradeTime = +response.data.data[response.data.data.length - 1].ts - const trades = response.data.data .map(trade => this.formatTrade(trade)) .filter( - a => a.timestamp >= range.from + 1 && a.timestamp < range.to + a => { + return a.timestamp >= range.from + 1 && a.timestamp <= range.to + } ) if (trades.length) { @@ -259,8 +316,7 @@ class Okex extends Exchange { earliestTradeTime >= range.from ) { console.log( - `[${this.id}.recoverMissingTrades] +${trades.length} ${ - range.pair + `[${this.id}.recoverMissingTrades] +${trades.length} ${range.pair } ... but theres more (${getHms(remainingMissingTime)} remaining)` ) return this.waitBeforeContinueRecovery().then(() => @@ -268,8 +324,7 @@ class Okex extends Exchange { ) } else { console.log( - `[${this.id}.recoverMissingTrades] +${trades.length} ${ - range.pair + `[${this.id}.recoverMissingTrades] +${trades.length} ${range.pair } (${getHms(remainingMissingTime)} remaining)` ) } @@ -286,87 +341,6 @@ class Okex extends Exchange { return totalRecovered }) } - - getLiquidationEndpoint(productId) { - const productType = this.types[productId].toUpperCase() - let endpoint = `https://www.okex.com/api/v5/public/liquidation-orders?instId=${productId}&instType=${productType}&uly=${productId - .split('-') - .slice(0, 2) - .join('-')}&state=filled` - - if (productType === 'FUTURES') { - endpoint += `&alias=${this.aliases[productId]}` - } - - return endpoint - } - - fetchLatestLiquidations() { - const productId = - this.liquidationProducts[ - this._liquidationProductIndex++ % this.liquidationProducts.length - ] - const endpoint = this.getLiquidationEndpoint(productId) - - this._liquidationAxiosHandler && this._liquidationAxiosHandler.cancel() - this._liquidationAxiosHandler = axios.CancelToken.source() - - axios - .get(endpoint) - .then(response => { - if ( - !this.apis.length || - !response.data || - response.data.msg || - !response.data.data || - !response.data.data.length || - !response.data.data[0] - ) { - //throw new Error(response.data ? response.data.msg : 'empty REST /liquidation-order') - return - } - - const liquidations = response.data.data[0].details.filter( - liquidation => { - return ( - !this.liquidationProductsReferences[productId] || - liquidation.ts > this.liquidationProductsReferences[productId] - ) - } - ) - - if (!liquidations.length) { - return - } - - this.liquidationProductsReferences[productId] = +liquidations[0].ts - - this.emitLiquidations( - null, - liquidations.map(liquidation => - this.formatLiquidation(liquidation, productId) - ) - ) - }) - .catch(err => { - let message = `[okex.fetchLatestLiquidations] ${productId} ${err.message} at ${endpoint}` - - if (err.response && err.response.data && err.response.data.msg) { - message += '\n\t' + err.response.data.msg - } - - console.error(message, `(instId: ${productId}})`) - - if (axios.isCancel(err)) { - return - } - - return err - }) - .then(() => { - delete this._liquidationAxiosHandler - }) - } } module.exports = Okex diff --git a/src/services/catalog.js b/src/services/catalog.js index a4ca136..30902e7 100644 --- a/src/services/catalog.js +++ b/src/services/catalog.js @@ -200,7 +200,7 @@ module.exports.parseMarket = function (market, noStable = true) { let localSymbolAlpha = localSymbol.replace(/[-_/:]/, '') let match - if (exchangeId !== 'BINANCE') { + if (!/BINANCE/.test(exchangeId)) { match = localSymbol.match(currencyPairLookup) } diff --git a/src/storage/influx.js b/src/storage/influx.js index 80378be..624a024 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -5,6 +5,7 @@ const config = require('../config') const socketService = require('../services/socket') const alertService = require('../services/alert') const { connections, updateIndexes } = require('../services/connections') +const { parseMarket } = require('../services/catalog') require('../typedef') @@ -839,9 +840,26 @@ class InfluxStorage { let query = `SELECT * FROM "${config.influxDatabase}"."${config.influxRetentionPrefix}${timeframeLitteral}"."trades_${timeframeLitteral}" WHERE time >= ${from}ms AND time < ${to}ms` if (markets.length) { - query += ` AND (${markets - .map(market => `market = '${market}'`) - .join(' OR ')})` + query += ` AND (${markets.map((marketOrIndex) => { + if (config.influxCollectors && marketOrIndex.indexOf(':') === -1 && socketService.serverSocket) { + const collector = socketService.getNodeByMarket(marketOrIndex) + + if (collector) { + const markets = collector.markets.filter(market => { + const product = parseMarket(market) + if (product.local === marketOrIndex) { + return true + } + }) + + if (markets.length) { + return markets.map(a => `market = '${a}'`).join(' OR ') + } + } + } + + return `market = '${marketOrIndex}'` + }).join(' OR ')})` } return this.influx From dc68b9fe771b9ff4e74ee757079222109fac8ee8 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Tue, 10 Oct 2023 15:21:23 +0200 Subject: [PATCH 13/20] remove trade queue --- src/config.js | 40 +----- src/exchange.js | 91 +++--------- src/exchanges/binance.js | 1 + src/server.js | 300 +-------------------------------------- 4 files changed, 27 insertions(+), 405 deletions(-) diff --git a/src/config.js b/src/config.js index 6b5d49d..ec0ddae 100644 --- a/src/config.js +++ b/src/config.js @@ -85,27 +85,11 @@ const defaultConfig = { // bypass origin restriction for given ips (comma separated) whitelist: [], - // enable websocket server (if you only use this for storing trade data set to false) - broadcast: false, - - // separate the broadcasts by n ms (0 = broadcast instantly) - broadcastDebounce: 0, - - // aggregate trades that came within same millisecond before broadcast - // (note) saving to storage is NOT impacted - // (warning) will add +50ms delay for confirmation that trade actually came on same ms - broadcastAggr: true, - - // will only broadcast trades >= broadcastThreshold - // expressed in base currency (ex: BTC) - // default 0 - broadcastThreshold: 0, - // enable api (historical/{from in ms}/{to in ms}/{timesfame in ms}/{markets separated by +}) api: true, // storage solution, either - // false | null (no storage, everything is wiped out after broadcast) + // false | null (no storage) // "files" (periodical text file), // "influx" (timeserie database), @@ -359,12 +343,6 @@ if (config.exchanges && typeof config.exchanges === 'string') { .filter(a => a.length) } -if (!config.api && config.broadcast) { - console.warn( - `[warning!] websocket is enabled but api is set to ${config.api}\n\t(ws server require an http server for the initial upgrade handshake)` - ) -} - if (!config.storage && config.collect) { console.warn( `[warning!] server will not persist any of the data it is receiving` @@ -375,21 +353,9 @@ if (!config.collect && !config.api) { console.warn(`[warning!] server has no purpose`) } -if (!config.storage && !config.collect && (config.broadcast || config.api)) { - console.warn( - `[warning!] ${ - config.broadcast && config.api - ? 'ws and api are' - : config.broadcast - ? 'ws is' - : 'api is' - } enabled but neither storage or collect is enabled (may be useless)` - ) -} - -if (config.broadcast && !config.collect) { +if (!config.storage && !config.collect && config.api) { console.warn( - `[warning!] collect is disabled but broadcast is set to ${config.broadcast} (may be useless)` + `[warning!] api is enabled but neither storage or collect is enabled (may be useless)` ) } diff --git a/src/exchange.js b/src/exchange.js index 349db07..c0f653f 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -73,23 +73,11 @@ class Exchange extends EventEmitter { */ this.maxConnectionsPerApi = 50 - /** - * Define if the incoming trades should be queued - * @type {boolean} - */ - this.shouldQueueTrades = false - /** * Pending recovery ranges * @type {{pair: string, from: number, to: number}[]} */ this.recoveryRanges = [] - - /** - * Trades goes theres while we wait for historical response - * @type {Trade[]} - */ - this.queuedTrades = [] } /** @@ -262,7 +250,7 @@ class Exchange extends EventEmitter { return } - if (!/ping|pong/.test(data)) { + if (!/ping|pong/i.test(data)) { console.debug( `[${this.id}.createWs] sending ${data.substr(0, 64)}${ data.length > 64 ? '...' : '' @@ -547,12 +535,24 @@ class Exchange extends EventEmitter { this.recoveryRanges.push(range) if (!recovering[this.id]) { + console.log(`[${this.id}.registerRangeForRecovery] exchange isn't recovering yet -> start recovering`) this.recoverNextRange() } } async recoverNextRange(sequencial) { if (!this.recoveryRanges.length || (recovering[this.id] && !sequencial)) { + if (recovering[this.id] && !sequencial) { + console.log(`[${this.id}] attempted to start manual recovery while already recovering`) + } + if (!this.recoveryRanges.length) { + console.log(`[${this.id}] no more range to recover`) + if (sequencial) { + console.log(`[${this.id}] set recovering[this.id] to false`) + delete recovering[this.id] + + } + } return } @@ -609,7 +609,7 @@ class Exchange extends EventEmitter { +range.to ).toISOString()})` } - connection.timestamp = range.to + connection.timestamp = Math.max(connection.timestamp, range.to) } // in rare case of slow recovery and fast reconnection happening, propagate to pending ranges for that pair @@ -637,31 +637,9 @@ class Exchange extends EventEmitter { } if (!this.recoveryRanges.length) { - console.log(`[${this.id}] no more ranges to recover`) + console.log(`[${this.id}] no more ranges to recover (delete recovering[this.id])`) delete recovering[this.id] - - if (this.queuedTrades.length) { - const sortedQueuedTrades = this.queuedTrades.sort( - (a, b) => a.timestamp - b.timestamp - ) - - console.log( - `[${this.id}] release trades queue (${ - sortedQueuedTrades.length - } trades, ${new Date(+sortedQueuedTrades[0].timestamp) - .toISOString() - .split('T') - .pop()} to ${new Date( - +sortedQueuedTrades[sortedQueuedTrades.length - 1].timestamp - ) - .toISOString() - .split('T') - .pop()})` - ) - this.emit('trades', sortedQueuedTrades) - this.queuedTrades = [] - } } else { return this.waitBeforeContinueRecovery().then(() => this.recoverNextRange(true) @@ -830,8 +808,6 @@ class Exchange extends EventEmitter { * @param {string[]} pairs pairs attached to ws at opening */ async onOpen(event, api) { - this.queueNextTrades() - const pairs = [...api._pending, ...api._connected] console.debug( @@ -956,11 +932,11 @@ class Exchange extends EventEmitter { * @param {Trade[]} trades */ emitTrades(source, trades) { - if (source && this.promisesOfApiReconnections[source]) { - return + if (!trades || !trades.length) { + return null } - this.queueControl(source, trades, 'trades') + this.emit('trades', trades, source) return true } @@ -971,26 +947,13 @@ class Exchange extends EventEmitter { * @param {Trade[]} trades */ emitLiquidations(source, trades) { - if (source && this.promisesOfApiReconnections[source]) { - return - } - - this.queueControl(source, trades, 'liquidations') - - return true - } - - queueControl(source, trades, type) { if (!trades || !trades.length) { return null } - if (this.shouldQueueTrades || recovering[this.id]) { - Array.prototype.push.apply(this.queuedTrades, trades) - return null - } + this.emit('liquidations', trades, source) - this.emit(type, trades, source) + return true } startKeepAlive(api, payload = { event: 'ping' }, every = 30000) { @@ -1110,20 +1073,6 @@ class Exchange extends EventEmitter { return true } - queueNextTrades(duration = 100) { - this.shouldQueueTrades = true - - if (this._unlockTimeout) { - clearTimeout(this._unlockTimeout) - } - - this._unlockTimeout = setTimeout(() => { - this._unlockTimeout = null - - this.shouldQueueTrades = false - }, duration) - } - /** * Wait between requests to prevent 429 HTTP errors * @returns {Promise} diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index 83d71d4..a983946 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -8,6 +8,7 @@ class Binance extends Exchange { this.id = 'BINANCE' this.lastSubscriptionId = 0 + this.maxConnectionsPerApi = 16 this.subscriptions = {} this.endpoints = { diff --git a/src/server.js b/src/server.js index f36d7eb..ec4457e 100644 --- a/src/server.js +++ b/src/server.js @@ -45,39 +45,11 @@ class Server extends EventEmitter { */ this.chunk = [] - /** - * delayedForBroadcast trades ready to be broadcasted next interval (see _broadcastDelayedTradesInterval) - * @type Trade[] - */ - this.delayedForBroadcast = [] - - /** - * active trades aggregations - * @type {{[aggrIdentifier: string]: Trade}} - */ - this.aggregating = {} - - /** - * already aggregated trades ready to be broadcasted (see _broadcastAggregatedTradesInterval) - * @type Trade[] - */ - this.aggregated = [] - this.BANNED_IPS = [] if (config.collect) { console.log( `\n[server] collect is enabled`, - config.broadcast && config.broadcastAggr - ? '\n\twill aggregate every trades that came on same ms (impact only broadcast)' - : '', - config.broadcast && config.broadcastDebounce && !config.broadcastAggr - ? `\n\twill broadcast trades every ${config.broadcastDebounce}ms` - : config.broadcast - ? `\n\twill broadcast ${ - config.broadcastAggr ? 'aggregated ' : '' - }trades instantly` - : '' ) console.log(`\tconnect to -> ${this.exchanges.map(a => a.id).join(', ')}`) @@ -107,10 +79,10 @@ class Server extends EventEmitter { } } - if (config.api || config.broadcast) { + if (config.api) { if (!config.port) { console.error( - `\n[server] critical error occured\n\t-> setting a network port is mandatory for API or broadcasting (value is ${config.port})\n\n` + `\n[server] critical error occured\n\t-> setting a network port is mandatory for API (value is ${config.port})\n\n` ) process.exit() } @@ -124,17 +96,6 @@ class Server extends EventEmitter { ) } - if (config.broadcast) { - this.createWSServer() - - if (config.broadcastAggr) { - this._broadcastAggregatedTradesInterval = setInterval( - this.broadcastAggregatedTrades.bind(this), - 50 - ) - } - } - // update banned users this.listenBannedIps() }) @@ -243,17 +204,8 @@ class Server extends EventEmitter { handleExchangesEvents() { this.exchanges.forEach(exchange => { - if (config.broadcast && config.broadcastAggr) { - exchange.on( - 'trades', - this.dispatchAggregateTrade.bind(this, exchange.id) - ) - } else { - exchange.on('trades', this.dispatchRawTrades.bind(this)) - } - + exchange.on('trades', this.dispatchRawTrades.bind(this)) exchange.on('liquidations', this.dispatchRawTrades.bind(this)) - exchange.on('disconnected', (pair, apiId, apiLength) => { const id = exchange.id + ':' + pair @@ -302,21 +254,6 @@ class Server extends EventEmitter { socketService.syncMarkets() }) - exchange.on('open', (apiId, pairs) => { - this.broadcastJson({ - type: 'exchange_connected', - id: exchange.id - }) - }) - - exchange.on('error', (apiId, message) => { - this.broadcastJson({ - type: 'exchange_error', - id: exchange.id, - message: message - }) - }) - exchange.on('close', (apiId, pairs, event) => { if (pairs.length) { console.error( @@ -331,122 +268,6 @@ class Server extends EventEmitter { this.reconnectApis([apiId], 'unexpected close') }, 1000) } - - this.broadcastJson({ - type: 'exchange_disconnected', - id: exchange.id - }) - }) - }) - } - - createWSServer() { - if (!config.broadcast) { - return - } - - this.wss = new WebSocket.Server({ - server: this.server - }) - - this.wss.on('listening', () => { - console.log( - `[server] websocket server listening at localhost:${config.port}` - ) - }) - - this.wss.on('connection', (ws, req) => { - const user = getIp(req) - const pairs = parsePairsFromWsRequest(req) - - if (pairs && pairs.length) { - ws.pairs = pairs - } - - const data = { - type: 'welcome', - supportedPairs: getActiveConnections(), - timestamp: Date.now(), - exchanges: this.exchanges.map(exchange => { - return { - id: exchange.id, - connected: exchange.apis.reduce((pairs, api) => { - pairs.concat(api._connected) - return pairs - }, []) - } - }) - } - - console.log( - `[${user}/ws/${ws.pairs.join('+')}] joined ${req.url} from ${ - req.headers['origin'] - }` - ) - - this.emit('connections', this.wss.clients.size) - - ws.send(JSON.stringify(data)) - - ws.on('message', event => { - const message = event.trim() - - const pairs = message.length - ? message - .split('+') - .map(a => a.trim()) - .filter(a => a.length) - : [] - - console.log(`[${user}/ws] subscribe to ${pairs.join(' + ')}`) - - ws.pairs = pairs - }) - - ws.on('close', event => { - let error = null - - switch (event) { - case 1002: - error = 'Protocol Error' - break - case 1003: - error = 'Unsupported Data' - break - case 1007: - error = 'Invalid frame payload data' - break - case 1008: - error = 'Policy Violation' - break - case 1009: - error = 'Message too big' - break - case 1010: - error = 'Missing Extension' - break - case 1011: - error = 'Internal Error' - break - case 1012: - error = 'Service Restart' - break - case 1013: - error = 'Try Again Later' - break - case 1014: - error = 'Bad Gateway' - break - case 1015: - error = 'TLS Handshake' - break - } - - if (error) { - console.log(`[${user}] unusual close "${error}"`) - } - - setTimeout(() => this.emit('connections', this.wss.clients.size), 100) }) }) } @@ -713,18 +534,6 @@ class Server extends EventEmitter { exchange.getProductsAndConnect(exchangePairs) } - - if (config.broadcast && config.broadcastDebounce && !config.broadcastAggr) { - this._broadcastDelayedTradesInterval = setInterval(() => { - if (!this.delayedForBroadcast.length) { - return - } - - this.broadcastTrades(this.delayedForBroadcast) - - this.delayedForBroadcast = [] - }, config.broadcastDebounce || 1000) - } } async connect(markets) { @@ -883,38 +692,6 @@ class Server extends EventEmitter { }) } - broadcastJson(data) { - if (!this.wss) { - return - } - - this.wss.clients.forEach(client => { - if (client.readyState === WebSocket.OPEN) { - client.send(JSON.stringify(data)) - } - }) - } - - broadcastTrades(trades) { - if (!this.wss) { - return - } - - const groups = groupTrades(trades, true, config.broadcastThreshold) - - this.wss.clients.forEach(client => { - if (client.readyState === WebSocket.OPEN) { - for (let i = 0; i < client.pairs.length; i++) { - if (groups[client.pairs[i]]) { - client.send( - JSON.stringify([client.pairs[i], groups[client.pairs[i]]]) - ) - } - } - } - }) - } - reconnectApis(apiIds, reason) { for (let exchange of this.exchanges) { for (let api of exchange.apis) { @@ -1009,77 +786,6 @@ class Server extends EventEmitter { this.chunk.push(trade) } } - - if (config.broadcast) { - if (!config.broadcastAggr && !config.broadcastDebounce) { - this.broadcastTrades(trades) - } else { - Array.prototype.push.apply(this.delayedForBroadcast, trades) - } - } - } - - dispatchAggregateTrade(exchange, data, apiId) { - const now = Date.now() - const length = data.length - - for (let i = 0; i < length; i++) { - const trade = data[i] - const identifier = exchange + ':' + trade.pair - - if (!trade.liquidation) { - // ping connection - connections[identifier].hit++ - connections[identifier].timestamp = trade.timestamp - } - - // save trade - if (this.storages) { - this.chunk.push(trade) - } - - if (this.aggregating[identifier]) { - const queuedTrade = this.aggregating[identifier] - - if ( - queuedTrade.timestamp === trade.timestamp && - queuedTrade.side === trade.side - ) { - queuedTrade.size += trade.size - queuedTrade.price += trade.price * trade.size - continue - } else { - queuedTrade.price /= queuedTrade.size - this.aggregated.push(queuedTrade) - } - } - - this.aggregating[identifier] = Object.assign({}, trade) - this.aggregating[identifier].timeout = now + 50 - this.aggregating[identifier].price *= this.aggregating[identifier].size - } - } - - broadcastAggregatedTrades() { - const now = Date.now() - - const onGoingAggregation = Object.keys(this.aggregating) - - for (let i = 0; i < onGoingAggregation.length; i++) { - const trade = this.aggregating[onGoingAggregation[i]] - if (now > trade.timeout) { - trade.price /= trade.size - this.aggregated.push(trade) - - delete this.aggregating[onGoingAggregation[i]] - } - } - - if (this.aggregated.length) { - this.broadcastTrades(this.aggregated) - - this.aggregated.splice(0, this.aggregated.length) - } } monitorExchangesActivity() { From f4c1173522197d60b1e02a837d06f4685ff60167 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Tue, 10 Oct 2023 15:47:53 +0200 Subject: [PATCH 14/20] security: fixes warnings npm audit --- package-lock.json | 1007 ++++++++++++++++++++------------------------- 1 file changed, 438 insertions(+), 569 deletions(-) diff --git a/package-lock.json b/package-lock.json index d79c051..4561174 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,9 +45,9 @@ } }, "node_modules/@opencensus/core/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", "bin": { "semver": "bin/semver" } @@ -89,9 +89,9 @@ } }, "node_modules/@opencensus/propagation-b3/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", "bin": { "semver": "bin/semver" } @@ -106,9 +106,9 @@ } }, "node_modules/@pm2/agent": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@pm2/agent/-/agent-2.0.1.tgz", - "integrity": "sha512-QKHMm6yexcvdDfcNE7PL9D6uEjoQPGRi+8dh+rc4Hwtbpsbh5IAvZbz3BVGjcd4HaX6pt2xGpOohG7/Y2L4QLw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@pm2/agent/-/agent-2.0.3.tgz", + "integrity": "sha512-xkqqCoTf5VsciMqN0vb9jthW7olVAi4KRFNddCc7ZkeJZ3i8QwZANr4NSH2H5DvseRFHq7MiPspRY/EWAFWWTg==", "dependencies": { "async": "~3.2.0", "chalk": "~3.0.0", @@ -120,8 +120,8 @@ "nssocket": "0.6.0", "pm2-axon": "~4.0.1", "pm2-axon-rpc": "~0.7.0", - "proxy-agent": "~5.0.0", - "semver": "~7.2.0", + "proxy-agent": "~6.3.0", + "semver": "~7.5.0", "ws": "~7.4.0" } }, @@ -151,17 +151,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/@pm2/agent/node_modules/semver": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.2.3.tgz", - "integrity": "sha512-utbW9Z7ZxVvwiIWkdOMLOR9G/NFXh2aRucghkVrEMJWuC++r3lCkBC3LwqBinyHzGMAJxY5tn6VakZGHObq5ig==", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@pm2/agent/node_modules/ws": { "version": "7.4.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", @@ -183,9 +172,9 @@ } }, "node_modules/@pm2/io": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.0.tgz", - "integrity": "sha512-3rToDVJaRoob5Lq8+7Q2TZFruoEkdORxwzFpZaqF4bmH6Bkd7kAbdPrI/z8X6k1Meq5rTtScM7MmDgppH6aLlw==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.2.tgz", + "integrity": "sha512-XAvrNoQPKOyO/jJyCu8jPhLzlyp35MEf7w/carHXmWKddPzeNOFSEpSEqMzPDawsvpxbE+i918cNN+MwgVsStA==", "dependencies": { "@opencensus/core": "0.0.9", "@opencensus/propagation-b3": "0.0.8", @@ -193,7 +182,7 @@ "debug": "~4.3.1", "eventemitter2": "^6.3.1", "require-in-the-middle": "^5.0.0", - "semver": "6.3.0", + "semver": "~7.5.4", "shimmer": "^1.2.0", "signal-exit": "^3.0.3", "tslib": "1.9.3" @@ -236,14 +225,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/@pm2/io/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "bin": { - "semver": "bin/semver.js" - } - }, "node_modules/@pm2/js-api": { "version": "0.6.7", "resolved": "https://registry.npmjs.org/@pm2/js-api/-/js-api-0.6.7.tgz", @@ -350,45 +331,23 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/@tootallnate/once": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", - "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", - "engines": { - "node": ">= 6" - } + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" }, "node_modules/accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "dependencies": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" }, "engines": { "node": ">= 0.6" } }, - "node_modules/acorn": { - "version": "8.9.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.9.0.tgz", - "integrity": "sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", - "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -509,9 +468,9 @@ } }, "node_modules/ast-types/node_modules/tslib": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", - "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/async": { "version": "3.2.4", @@ -536,9 +495,9 @@ } }, "node_modules/async-listener/node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", "bin": { "semver": "bin/semver" } @@ -563,6 +522,14 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/basic-ftp": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.3.tgz", + "integrity": "sha512-QHX8HLlncOLpy54mh+k/sWIFd0ThmRqwe9ZjELybGZK+tZ8rUb9VO0saKJUROTbE+KhzDUT7xziGpGrW8Kmd+g==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -593,23 +560,26 @@ "integrity": "sha512-Ylo+MAo5BDUq1KA3f3R/MFhh+g8cnHmo8bz3YPGhI1znrMaf77ol1sfvYJzsw3nTE+Y2GryfDxBaR+AqpAkEHQ==" }, "node_modules/body-parser": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.1.tgz", - "integrity": "sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "dependencies": { - "bytes": "3.1.1", - "content-type": "~1.0.4", + "bytes": "3.1.2", + "content-type": "~1.0.5", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.8.1", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.9.6", - "raw-body": "2.4.2", - "type-is": "~1.6.18" + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" } }, "node_modules/body-parser/node_modules/debug": { @@ -620,50 +590,6 @@ "ms": "2.0.0" } }, - "node_modules/body-parser/node_modules/http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/body-parser/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "node_modules/body-parser/node_modules/qs": { - "version": "6.9.6", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz", - "integrity": "sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ==", - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/body-parser/node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" - }, - "node_modules/body-parser/node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "engines": { - "node": ">=0.6" - } - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -707,13 +633,25 @@ } }, "node_modules/bytes": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", - "integrity": "sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "engines": { "node": ">= 0.8" } }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/chalk": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", @@ -806,20 +744,20 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "node_modules/content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "dependencies": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" }, "engines": { "node": ">= 0.6" } }, "node_modules/content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", "engines": { "node": ">= 0.6" } @@ -834,9 +772,9 @@ } }, "node_modules/cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", "engines": { "node": ">= 0.6" } @@ -846,11 +784,6 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" - }, "node_modules/cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -883,11 +816,11 @@ } }, "node_modules/data-uri-to-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", - "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.1.tgz", + "integrity": "sha512-MZd3VlchQkp8rdend6vrx7MmVDJzSNTBvghvKjirLkD+WTChA3KUf0jkE68Q4UyctNqI11zZO9/x2Yx+ub5Cvg==", "engines": { - "node": ">= 6" + "node": ">= 14" } }, "node_modules/dayjs": { @@ -919,23 +852,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" - }, "node_modules/degenerator": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-3.0.4.tgz", - "integrity": "sha512-Z66uPeBfHZAHVmue3HPfyKu2Q0rC2cRxbTOsvmU/po5fvvcx27W4mIu9n0PUlQih4oUYvcG1BsbtVv8x7KDOSw==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", "dependencies": { - "ast-types": "^0.13.2", - "escodegen": "^1.8.1", - "esprima": "^4.0.0", - "vm2": "^3.9.17" + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" }, "engines": { - "node": ">= 6" + "node": ">= 14" } }, "node_modules/delayed-stream": { @@ -947,17 +874,21 @@ } }, "node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", @@ -970,7 +901,7 @@ "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/emitter-listener": { "version": "1.1.2", @@ -983,7 +914,7 @@ "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", "engines": { "node": ">= 0.8" } @@ -1035,7 +966,7 @@ "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "node_modules/escape-string-regexp": { "version": "4.0.0", @@ -1049,21 +980,20 @@ } }, "node_modules/escodegen": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", - "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", "dependencies": { "esprima": "^4.0.1", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" + "estraverse": "^5.2.0", + "esutils": "^2.0.2" }, "bin": { "escodegen": "bin/escodegen.js", "esgenerate": "bin/esgenerate.js" }, "engines": { - "node": ">=4.0" + "node": ">=6.0" }, "optionalDependencies": { "source-map": "~0.6.1" @@ -1082,9 +1012,9 @@ } }, "node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "engines": { "node": ">=4.0" } @@ -1100,7 +1030,7 @@ "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "engines": { "node": ">= 0.6" } @@ -1111,37 +1041,38 @@ "integrity": "sha512-5EM1GHXycJBS6mauYAbVKT1cVs7POKWb2NXD4Vyt8dDqeZa7LaDK1/sjtL+Zb0lzTpSNil4596Dyu97hz37QLg==" }, "node_modules/express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", "dependencies": { - "accepts": "~1.3.7", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.5.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "~1.1.2", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" @@ -1156,31 +1087,26 @@ "integrity": "sha512-nE96xaxGfxiS5jP3tD3kIW1Jg9yQgX0rXCs3rCkZtmbWHEGyotwaezkLj7bnB41Z0uaOLM8W4AX6qHao4IZ2YA==" }, "node_modules/express/node_modules/body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", "dependencies": { - "bytes": "3.1.0", + "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "engines": { - "node": ">= 0.8" - } - }, - "node_modules/express/node_modules/bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", - "engines": { - "node": ">= 0.8" + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" } }, "node_modules/express/node_modules/debug": { @@ -1192,12 +1118,12 @@ } }, "node_modules/express/node_modules/raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", "dependencies": { - "bytes": "3.1.0", - "http-errors": "1.7.2", + "bytes": "3.1.2", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", "unpipe": "1.0.0" }, @@ -1223,24 +1149,11 @@ "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz", "integrity": "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==" }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" - }, "node_modules/fclone": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz", "integrity": "sha512-GDqVQezKzRABdeqflsgMr7ktzgF9CyS+p2oe0jJqUY6izSSbhPIQJDpoU4PtGcD7VPM9xh/dVrTu6z1nwgmEGw==" }, - "node_modules/file-uri-to-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-2.0.0.tgz", - "integrity": "sha512-hjPFI8oE/2iQPVe4gbrJ73Pp+Xfub2+WI2LlXDbsaJBwT5wuMh35WNWVYYTpnz895shtwfyutMFLFywpQAFdLg==", - "engines": { - "node": ">= 6" - } - }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1253,16 +1166,16 @@ } }, "node_modules/finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", - "statuses": "~1.5.0", + "statuses": "2.0.1", "unpipe": "~1.0.0" }, "engines": { @@ -1310,9 +1223,9 @@ } }, "node_modules/forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "engines": { "node": ">= 0.6" } @@ -1320,7 +1233,7 @@ "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "engines": { "node": ">= 0.6" } @@ -1343,37 +1256,37 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "node_modules/ftp": { - "version": "0.3.10", - "resolved": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz", - "integrity": "sha512-faFVML1aBx2UoDStmLwv2Wptt4vw5x03xxX172nhA5Y5HBshW5JweqQ2W4xL4dezQTG8inJsuYcpPHHU3X5OTQ==", - "dependencies": { - "readable-stream": "1.1.x", - "xregexp": "2.0.0" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, + "node_modules/get-intrinsic": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", + "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-uri": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-3.0.2.tgz", - "integrity": "sha512-+5s0SJbGoyiJTZZ2JTpFPLMPSch72KEqGOTvQsBqg0RBWvwhWUSYZFAtz3TPW0GXJuLBJPts1E241iHg+VRfhg==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.2.tgz", + "integrity": "sha512-5KLucCJobh8vBY1K07EFV4+cPZH3mrV9YeAruUseCQKHB58SGjjT2l9/eA9LD082IiuMjSlFJEcdJ27TXvbZNw==", "dependencies": { - "@tootallnate/once": "1", - "data-uri-to-buffer": "3", - "debug": "4", - "file-uri-to-path": "2", - "fs-extra": "^8.1.0", - "ftp": "^0.3.10" + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.0", + "debug": "^4.3.4", + "fs-extra": "^8.1.0" }, "engines": { - "node": ">= 6" + "node": ">= 14" } }, "node_modules/get-uri/node_modules/debug": { @@ -1461,6 +1374,28 @@ "node": ">=8" } }, + "node_modules/has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/http_ece": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/http_ece/-/http_ece-1.1.0.tgz", @@ -1473,31 +1408,41 @@ } }, "node_modules/http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" }, "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/http-proxy-agent": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", - "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.0.tgz", + "integrity": "sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==", "dependencies": { - "@tootallnate/once": "1", - "agent-base": "6", - "debug": "4" + "agent-base": "^7.1.0", + "debug": "^4.3.4" }, "engines": { - "node": ">= 6" + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" } }, "node_modules/http-proxy-agent/node_modules/debug": { @@ -1588,9 +1533,9 @@ "integrity": "sha512-QQU9CgwnaEV6zMrK8+vhVItsdoKFqDioXJrjJhRQaff9utvT3N0jcrQJT9qnxFLktqgJ5ngbDY68Zh4eo4uD/w==" }, "node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ini": { "version": "1.3.8", @@ -1664,11 +1609,6 @@ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, - "node_modules/isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" - }, "node_modules/js-git": { "version": "0.7.8", "resolved": "https://registry.npmjs.org/js-git/-/js-git-0.7.8.tgz", @@ -1736,18 +1676,6 @@ "node": ">=0.2.0" } }, - "node_modules/levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", - "dependencies": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -1762,11 +1690,11 @@ } }, "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dependencies": { - "yallist": "^3.0.2" + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" } }, "node_modules/media-typer": { @@ -1802,19 +1730,19 @@ } }, "node_modules/mime-db": { - "version": "1.46.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.29", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", - "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dependencies": { - "mime-db": "1.46.0" + "mime-db": "1.52.0" }, "engines": { "node": ">= 0.6" @@ -1837,9 +1765,12 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/mkdirp": { "version": "1.0.4", @@ -1884,9 +1815,9 @@ } }, "node_modules/negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", "engines": { "node": ">= 0.6" } @@ -1947,10 +1878,18 @@ "node": ">=0.10.0" } }, + "node_modules/object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "dependencies": { "ee-first": "1.1.1" }, @@ -1966,39 +1905,33 @@ "wrappy": "1" } }, - "node_modules/optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "node_modules/pac-proxy-agent": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz", + "integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==", "dependencies": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "pac-resolver": "^7.0.0", + "socks-proxy-agent": "^8.0.2" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 14" } }, - "node_modules/pac-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-5.0.0.tgz", - "integrity": "sha512-CcFG3ZtnxO8McDigozwE3AqAw15zDvGH+OjXO4kzf7IkEKkQ4gxQ+3sdF50WmhQ4P/bVusXcqNE2S3XrNURwzQ==", + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", "dependencies": { - "@tootallnate/once": "1", - "agent-base": "6", - "debug": "4", - "get-uri": "3", - "http-proxy-agent": "^4.0.1", - "https-proxy-agent": "5", - "pac-resolver": "^5.0.0", - "raw-body": "^2.2.0", - "socks-proxy-agent": "5" + "debug": "^4.3.4" }, "engines": { - "node": ">= 8" + "node": ">= 14" } }, "node_modules/pac-proxy-agent/node_modules/debug": { @@ -2017,22 +1950,34 @@ } } }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/pac-proxy-agent/node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/pac-resolver": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-5.0.1.tgz", - "integrity": "sha512-cy7u00ko2KVgBAjuhevqpPeHIkCIqPe1v24cydhWjmeuzaBfmUWFCZJ1iAh5TuVzVZoUzXIW7K8sMYOZ84uZ9Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.0.tgz", + "integrity": "sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg==", "dependencies": { - "degenerator": "^3.0.2", - "ip": "^1.1.5", + "degenerator": "^5.0.0", + "ip": "^1.1.8", "netmask": "^2.0.2" }, "engines": { - "node": ">= 8" + "node": ">= 14" } }, "node_modules/pako": { @@ -2088,25 +2033,6 @@ "node": ">=10" } }, - "node_modules/pidusage/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/pm2": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/pm2/-/pm2-5.3.0.tgz", @@ -2290,26 +2216,6 @@ "node": ">=8" } }, - "node_modules/pm2-sysmonit/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "optional": true - }, "node_modules/pm2/node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -2331,14 +2237,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/prettier": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.2.tgz", @@ -2363,11 +2261,11 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dependencies": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { @@ -2375,21 +2273,32 @@ } }, "node_modules/proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-5.0.0.tgz", - "integrity": "sha512-gkH7BkvLVkSfX9Dk27W6TyNOWWZWRilRfk1XxGNWOYJ2TuedAv1yFpCaU9QSBmBe716XOTNpYNOzhysyw8xn7g==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.3.1.tgz", + "integrity": "sha512-Rb5RVBy1iyqOtNl15Cw/llpeLH8bsb37gM1FUfKQ+Wck6xHlbAhWGUFiTRHtkjqGTA5pSHz6+0hrPW/oECihPQ==", "dependencies": { - "agent-base": "^6.0.0", - "debug": "4", - "http-proxy-agent": "^4.0.0", - "https-proxy-agent": "^5.0.0", - "lru-cache": "^5.1.1", - "pac-proxy-agent": "^5.0.0", - "proxy-from-env": "^1.0.0", - "socks-proxy-agent": "^5.0.0" + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" }, "engines": { - "node": ">= 8" + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" } }, "node_modules/proxy-agent/node_modules/debug": { @@ -2408,6 +2317,18 @@ } } }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/proxy-agent/node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2419,11 +2340,17 @@ "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "node_modules/qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, "engines": { "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/range-parser": { @@ -2435,12 +2362,12 @@ } }, "node_modules/raw-body": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.2.tgz", - "integrity": "sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dependencies": { - "bytes": "3.1.1", - "http-errors": "1.8.1", + "bytes": "3.1.2", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", "unpipe": "1.0.0" }, @@ -2448,39 +2375,6 @@ "node": ">= 0.8" } }, - "node_modules/raw-body/node_modules/http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "node_modules/raw-body/node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" - }, - "node_modules/raw-body/node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "engines": { - "node": ">=0.6" - } - }, "node_modules/read": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/read/-/read-1.0.7.tgz", @@ -2492,17 +2386,6 @@ "node": ">=0.8" } }, - "node_modules/readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -2584,9 +2467,23 @@ ] }, "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, "node_modules/safer-buffer": { "version": "2.1.2", @@ -2599,9 +2496,9 @@ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "node_modules/semver": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", - "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -2629,23 +2526,23 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "dependencies": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "2.0.0", "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", + "ms": "2.1.3", + "on-finished": "2.4.1", "range-parser": "~1.2.1", - "statuses": "~1.5.0" + "statuses": "2.0.1" }, "engines": { "node": ">= 0.8.0" @@ -2662,37 +2559,50 @@ "node_modules/send/node_modules/debug/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/send/node_modules/ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "dependencies": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.18.0" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -2721,16 +2631,27 @@ } }, "node_modules/socks-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-5.0.1.tgz", - "integrity": "sha512-vZdmnjb9a2Tz6WEQVIurybSwElwPxMZaIc7PzqbJTrezcKNznv6giT7J7tZDZ1BojVaa1jvO/UiUdhDVB0ACoQ==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz", + "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==", "dependencies": { - "agent-base": "^6.0.2", - "debug": "4", - "socks": "^2.3.3" + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "socks": "^2.7.1" }, "engines": { - "node": ">= 6" + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" } }, "node_modules/socks-proxy-agent/node_modules/debug": { @@ -2782,18 +2703,13 @@ "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==" }, "node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, - "node_modules/string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2817,9 +2733,9 @@ } }, "node_modules/systeminformation": { - "version": "5.18.3", - "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.18.3.tgz", - "integrity": "sha512-k+gk7zSi0hI/m3Mgu1OzR8j9BfXMDYa2HUMBdEQZUVCVAO326kDrzrvtVMljiSoDs6T6ojI0AHneDn8AMa0Y6A==", + "version": "5.21.11", + "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.21.11.tgz", + "integrity": "sha512-dIJEGoP5W7k4JJGje/b+inJrOL5hV9LPsUi5ndBvJydI80CVEcu2DZYgt6prdRErDi2SA4SqYd/WMR4b+u34mA==", "optional": true, "os": [ "darwin", @@ -2854,9 +2770,9 @@ } }, "node_modules/toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "engines": { "node": ">=0.6" } @@ -2887,17 +2803,6 @@ "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" }, - "node_modules/type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", - "dependencies": { - "prelude-ls": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2997,21 +2902,6 @@ "lodash": "^4.17.14" } }, - "node_modules/vm2": { - "version": "3.9.19", - "resolved": "https://registry.npmjs.org/vm2/-/vm2-3.9.19.tgz", - "integrity": "sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg==", - "dependencies": { - "acorn": "^8.7.0", - "acorn-walk": "^8.2.0" - }, - "bin": { - "vm2": "bin/vm2" - }, - "engines": { - "node": ">=6.0" - } - }, "node_modules/web-push": { "version": "3.4.5", "resolved": "https://registry.npmjs.org/web-push/-/web-push-3.4.5.tgz", @@ -3055,14 +2945,6 @@ "ms": "2.0.0" } }, - "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3080,21 +2962,13 @@ } }, "node_modules/ws": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz", - "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.3.tgz", + "integrity": "sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA==", "dependencies": { "async-limiter": "~1.0.0" } }, - "node_modules/xregexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-2.0.0.tgz", - "integrity": "sha512-xl/50/Cf32VsGq/1R8jJE5ajH1yMCQkpmoS10QbFZWl2Oor4H0Me64Pu2yxvsRWK3m6soJbmGfzSR7BYmDcWAA==", - "engines": { - "node": "*" - } - }, "node_modules/yaeti": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", @@ -3103,11 +2977,6 @@ "node": ">=0.10.32" } }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" - }, "node_modules/yamljs": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/yamljs/-/yamljs-0.3.0.tgz", From 6ebef88b2201406e286436e5019098801d18c9f1 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Wed, 11 Oct 2023 02:48:57 +0200 Subject: [PATCH 15/20] non blocking recovery --- .gitignore | 1 + data/.gitkeep | 0 src/exchange.js | 155 ++++++++++++++---------------------- src/exchanges/okex.js | 12 ++- src/helper.js | 31 ++------ src/server.js | 10 +-- src/services/connections.js | 66 +++++++-------- src/storage/influx.js | 63 +++++++-------- 8 files changed, 141 insertions(+), 197 deletions(-) delete mode 100644 data/.gitkeep diff --git a/.gitignore b/.gitignore index f914609..6aca67f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ data/* *.config.json persistence.json coinalize.key +events.log .env \ No newline at end of file diff --git a/data/.gitkeep b/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/exchange.js b/src/exchange.js index c0f653f..c878c84 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -129,10 +129,9 @@ class Exchange extends EventEmitter { /** * Link exchange to a pair * @param {string} pair - * @param {boolean} returnConnectedEvent * @returns {Promise} */ - async link(pair, returnConnectedEvent) { + async link(pair) { pair = pair.replace(/[^:]*:/, '') if (!this.isMatching(pair)) { @@ -145,41 +144,45 @@ class Exchange extends EventEmitter { const api = await this.resolveApi(pair) - if (returnConnectedEvent) { - let promiseOfApiOpen + let promiseOfApiOpen - if (api && this.connecting[api.id]) { - // need to init new ws connection - promiseOfApiOpen = this.connecting[api.id].promise - } else { - // api already opened - promiseOfApiOpen = Promise.resolve(true) + if (api && this.connecting[api.id]) { + // need to init new ws connection + promiseOfApiOpen = this.connecting[api.id].promise + } else { + // api already opened + promiseOfApiOpen = Promise.resolve(true) + } + + try { + if (!(await promiseOfApiOpen)) { + return } + } catch (error) { + console.log('error while linking', error) + } - if (await promiseOfApiOpen) { - return new Promise(resolve => { - let timeout + return new Promise(resolve => { + let timeout - const connectedEventHandler = connectedPair => { - if (connectedPair === pair) { - clearTimeout(timeout) - this.off('connected', connectedEventHandler) + const connectedEventHandler = connectedPair => { + if (connectedPair === pair) { + clearTimeout(timeout) + this.off('connected', connectedEventHandler) - resolve() - } - } + resolve() + } + } - this.on('connected', connectedEventHandler) + this.on('connected', connectedEventHandler) - timeout = setTimeout(() => { - console.error( - `[${this.id}/link] ${pair} connected event never fired, resolving returnConnectedEvent immediately` - ) - connectedEventHandler(pair) - }, 10000) - }) - } - } + timeout = setTimeout(() => { + console.error( + `[${this.id}/link] ${pair} connected event never fired, resolving returnConnectedEvent immediately` + ) + connectedEventHandler(pair) + }, 10000) + }) } async resolveApi(pair) { @@ -252,8 +255,7 @@ class Exchange extends EventEmitter { if (!/ping|pong/i.test(data)) { console.debug( - `[${this.id}.createWs] sending ${data.substr(0, 64)}${ - data.length > 64 ? '...' : '' + `[${this.id}.createWs] sending ${data.substr(0, 64)}${data.length > 64 ? '...' : '' } to ${api.url}` ) } @@ -299,11 +301,7 @@ class Exchange extends EventEmitter { this.onApiCreated(api) resolve(api) } else { - reject( - new Error( - 'Failed to establish a websocket connection with exchange' - ) - ) + reject(new Error('Failed to establish a websocket connection with exchange')) } } }) @@ -313,8 +311,7 @@ class Exchange extends EventEmitter { async subscribePendingPairs(api) { console.debug( - `[${this.id}.subscribePendingPairs] subscribe to ${ - api._pending.length + `[${this.id}.subscribePendingPairs] subscribe to ${api._pending.length } pairs of api ${api.url} (${api._pending.join(', ')})` ) @@ -355,8 +352,7 @@ class Exchange extends EventEmitter { } console.debug( - `[${this.id}.unlink] disconnecting ${pair} ${ - skipSending ? '(skip sending)' : '' + `[${this.id}.unlink] disconnecting ${pair} ${skipSending ? '(skip sending)' : '' }` ) @@ -400,7 +396,7 @@ class Exchange extends EventEmitter { this.apis[i].url === url && (!this.maxConnectionsPerApi || this.apis[i]._connected.length + this.apis[i]._pending.length < - this.maxConnectionsPerApi) + this.maxConnectionsPerApi) ) { return this.apis[i] } @@ -469,8 +465,7 @@ class Exchange extends EventEmitter { api.onmessage = null console.debug( - `[${this.id}.reconnectApi] reconnect api ${api.id}${ - reason ? ' reason: ' + reason : '' + `[${this.id}.reconnectApi] reconnect api ${api.id}${reason ? ' reason: ' + reason : '' } (url: ${api.url}, _connected: ${api._connected.join( ', ' )}, _pending: ${api._pending.join(', ')})` @@ -495,8 +490,7 @@ class Exchange extends EventEmitter { pairsToReconnect ).then(() => { console.log( - `[${this.id}.reconnectApi] done reconnecting api (was ${api.id}${ - reason ? ' because of ' + reason : '' + `[${this.id}.reconnectApi] done reconnecting api (was ${api.id}${reason ? ' because of ' + reason : '' })` ) delete this.promisesOfApiReconnections[api.id] @@ -514,13 +508,9 @@ class Exchange extends EventEmitter { return } - if ( - !connection.forceRecovery && - connection.lastConnectionMissEstimate < 10 - ) { + if (connection.lastConnectionMissEstimate < 10) { + // too much chance to be zero recovery so we skip to avoid being 429'd because of those return - } else if (connection.forceRecovery) { - delete connection.forceRecovery } const now = Date.now() @@ -542,15 +532,11 @@ class Exchange extends EventEmitter { async recoverNextRange(sequencial) { if (!this.recoveryRanges.length || (recovering[this.id] && !sequencial)) { - if (recovering[this.id] && !sequencial) { - console.log(`[${this.id}] attempted to start manual recovery while already recovering`) - } if (!this.recoveryRanges.length) { console.log(`[${this.id}] no more range to recover`) if (sequencial) { - console.log(`[${this.id}] set recovering[this.id] to false`) + console.log(`[${this.id}] recoverNextRange was called sequentially yet no recoveryRanges are left to recover (impossible case)`) delete recovering[this.id] - } } return @@ -558,11 +544,14 @@ class Exchange extends EventEmitter { const range = this.recoveryRanges.shift() + const originalRange = { + from: range.from, + to: range.to, + } const missingTime = range.to - range.from console.log( - `[${this.id}.recoverTrades] get missing trades for ${ - range.pair + `[${this.id}.recoverTrades] get missing trades for ${range.pair } (expecting ${range.missEstimate} on a ${getHms( missingTime )} blackout going from ${new Date(range.from) @@ -571,8 +560,6 @@ class Exchange extends EventEmitter { .pop()} to ${new Date(range.to).toISOString().split('T').pop()})` ) - const connection = connections[this.id + ':' + range.pair] - recovering[this.id] = true try { @@ -580,8 +567,7 @@ class Exchange extends EventEmitter { if (recoveredCount) { console.info( - `[${this.id}.recoverTrades] recovered ${recoveredCount} (expected ${ - range.missEstimate + `[${this.id}.recoverTrades] recovered ${recoveredCount} (expected ${range.missEstimate }) trades on ${this.id}:${range.pair} (${getHms( missingTime - (range.to - range.from) )} recovered out of ${getHms(missingTime)} | ${getHms( @@ -590,42 +576,26 @@ class Exchange extends EventEmitter { ) } else { console.info( - `[${this.id}.recoverTrades] 0 trade recovered on ${ - range.pair + `[${this.id}.recoverTrades] 0 trade recovered on ${range.pair } (expected ${range.missEstimate} for ${getHms( missingTime )} blackout)` ) } - if (connection) { - // save new timestamp to connection - if (connection.timestamp && connection.timestamp > range.to) { - ;`[${this.id}.recoverTrades] ${ - range.pair - } trade recovery is late on the schedule (last emitted trade: ${new Date( - +connection.timestamp - ).toISOString()}, last recovered trade: ${new Date( - +range.to - ).toISOString()})` - } - connection.timestamp = Math.max(connection.timestamp, range.to) - } - - // in rare case of slow recovery and fast reconnection happening, propagate to pending ranges for that pair + // shrink overlapping ranges for (let i = 0; i < this.recoveryRanges.length; i++) { const nextRange = this.recoveryRanges[i] if (nextRange.pair === range.pair) { - const newFrom = Math.max(nextRange.from, range.from) - - if (nextRange.from !== newFrom) { - nextRange.from = Math.max(nextRange.from, range.from) - - if (nextRange.from > nextRange.to) { - this.recoveryRanges.splice(i--, 1) - continue - } + // range might have been created while this one was recovering + // ensure no dupes + nextRange.from = Math.max(nextRange.from, originalRange.to) + nextRange.to = Math.max(nextRange.to, originalRange.to) + + if (nextRange.from > nextRange.to) { + this.recoveryRanges.splice(i--, 1) + continue } } } @@ -637,7 +607,7 @@ class Exchange extends EventEmitter { } if (!this.recoveryRanges.length) { - console.log(`[${this.id}] no more ranges to recover (delete recovering[this.id])`) + console.log(`[${this.id}] no more range to recover`) delete recovering[this.id] } else { @@ -840,8 +810,6 @@ class Exchange extends EventEmitter { `[${this.id}.onError] ${pairs.join(',')}'s api errored`, event.message ) - - this.emit('error', api.id, event.message) } /** @@ -861,8 +829,7 @@ class Exchange extends EventEmitter { this.failedConnections++ const delay = 1000 * this.failedConnections console.debug( - `[${this.id}] api refused connection, sleeping ${ - delay / 1000 + `[${this.id}] api refused connection, sleeping ${delay / 1000 }s before trying again` ) await sleep(delay) diff --git a/src/exchanges/okex.js b/src/exchanges/okex.js index f66c048..935ce9b 100644 --- a/src/exchanges/okex.js +++ b/src/exchanges/okex.js @@ -273,14 +273,16 @@ class Okex extends Exchange { console.log(`[${this.id}.recoverMissingTrades] +${liquidations.length} liquidations for ${range.pair}`) if (liquidations.length) { - this.emitLiquidations(null, liquidations) + this.emitLiquidations(null, liquidations.map( + liquidation => this.formatLiquidation(liquidation, range.pair) + )) } } let endpoint if (beforeTradeId) { endpoint = `https://www.okex.com/api/v5/market/history-trades?instId=${range.pair - }&limit=500${beforeTradeId ? '&after=' + beforeTradeId : ''}` + }&limit=100${beforeTradeId ? '&after=' + beforeTradeId : ''}` } else { endpoint = `https://www.okex.com/api/v5/market/trades?instId=${range.pair}&limit=500` } @@ -294,12 +296,8 @@ class Okex extends Exchange { const earliestTradeTime = +response.data.data[response.data.data.length - 1].ts const trades = response.data.data + .filter(trade => Number(trade.ts) > range.from && (beforeTradeId || Number(trade.ts) < range.to)) .map(trade => this.formatTrade(trade)) - .filter( - a => { - return a.timestamp >= range.from + 1 && a.timestamp <= range.to - } - ) if (trades.length) { this.emitTrades(null, trades) diff --git a/src/helper.js b/src/helper.js index 6608d62..d965652 100644 --- a/src/helper.js +++ b/src/helper.js @@ -81,14 +81,10 @@ module.exports = { return output }, - groupTrades(trades, includeMarket, threshold = 0) { + groupTrades(trades) { const groups = {} for (let i = 0; i < trades.length; i++) { - if (trades[i].size < threshold) { - continue - } - const trade = trades[i] const identifier = trade.exchange + ':' + trade.pair @@ -96,25 +92,12 @@ module.exports = { groups[identifier] = [] } - let toPush - - if (includeMarket) { - toPush = [ - trade.exchange, - trade.pair, - trade.timestamp, - trade.price, - trade.size, - trade.side === 'buy' ? 1 : 0 - ] - } else { - toPush = [ - trade.timestamp, - trade.price, - trade.size, - trade.side === 'buy' ? 1 : 0 - ] - } + const toPush = [ + trade.timestamp, + trade.price, + trade.size, + trade.side === 'buy' ? 1 : 0 + ] if (trade.liquidation) { toPush.push(1) diff --git a/src/server.js b/src/server.js index ec4457e..34c2d39 100644 --- a/src/server.js +++ b/src/server.js @@ -1,5 +1,4 @@ const EventEmitter = require('events') -const WebSocket = require('ws') const fs = require('fs') const { getIp, @@ -24,7 +23,6 @@ const { recovering, updateConnectionStats, dumpConnections, - getActiveConnections } = require('./services/connections') class Server extends EventEmitter { @@ -255,17 +253,19 @@ class Server extends EventEmitter { }) exchange.on('close', (apiId, pairs, event) => { + const reason = event.reason || 'no reason' + if (pairs.length) { console.error( `[${exchange.id}] api closed unexpectedly (${apiId}, ${ event.code - }, ${event.reason || 'no reason'}) (was handling ${pairs.join( + }, ${reason}) (was handling ${pairs.join( ',' )})` ) setTimeout(() => { - this.reconnectApis([apiId], 'unexpected close') + this.reconnectApis([apiId], reason) }, 1000) } }) @@ -569,7 +569,7 @@ class Server extends EventEmitter { for (let market of exchangeMarkets) { try { - await exchange.link(market, true) + await exchange.link(market) config.pairs.push(market) results.push(`${market} ✅`) } catch (error) { diff --git a/src/services/connections.js b/src/services/connections.js index a29e10f..3bc7607 100644 --- a/src/services/connections.js +++ b/src/services/connections.js @@ -23,32 +23,32 @@ module.exports.recovering = {} */ const indexes = (module.exports.indexes = []) -;(module.exports.registerIndexes = function () { - indexes.splice(0, indexes.length) + ; (module.exports.registerIndexes = function () { + indexes.splice(0, indexes.length) - const cacheIndexes = {} + const cacheIndexes = {} - for (const market of config.pairs) { - const product = parseMarket(market) + for (const market of config.pairs) { + const product = parseMarket(market) - if (config.indexExchangeBlacklist.indexOf(product.exchange) !== -1 || config.indexQuoteWhitelist.indexOf(product.quote) === -1) { - continue - } + if (config.indexExchangeBlacklist.indexOf(product.exchange) !== -1 || config.indexQuoteWhitelist.indexOf(product.quote) === -1) { + continue + } - if (!cacheIndexes[product.local]) { - cacheIndexes[product.local] = { - id: product.local, - markets: [] + if (!cacheIndexes[product.local]) { + cacheIndexes[product.local] = { + id: product.local, + markets: [] + } } - } - cacheIndexes[product.local].markets.push(market) - } + cacheIndexes[product.local].markets.push(market) + } - for (const localPair in cacheIndexes) { - indexes.push(cacheIndexes[localPair]) - } -})() + for (const localPair in cacheIndexes) { + indexes.push(cacheIndexes[localPair]) + } + })() /** * Read the connections file and return the content @@ -228,20 +228,18 @@ module.exports.restoreConnections = async function () { } if ( - !persistance[market].forceRecovery && - (!persistance[market].timestamp || - (config.staleConnectionThreshold > 0 && - now - persistance[market].timestamp > - config.staleConnectionThreshold)) + !persistance[market].timestamp || + (config.staleConnectionThreshold > 0 && + now - persistance[market].timestamp > + config.staleConnectionThreshold) ) { console.log( - `[connections] couldn't restore ${market}'s connection because ${ - persistance[market].timestamp - ? `last ping is too old (${getHms( - now - persistance[market].timestamp, - true - )} ago)` - : `last ping is unknown` + `[connections] couldn't restore ${market}'s connection because ${persistance[market].timestamp + ? `last ping is too old (${getHms( + now - persistance[market].timestamp, + true + )} ago)` + : `last ping is unknown` }` ) // connection is to old (too much data to recover) @@ -370,8 +368,10 @@ module.exports.registerConnection = function (id, exchange, pair) { if (config.pairs.indexOf(id) === -1) { // force fetch last 1h30 of data through recent trades - connections[id].forceRecovery = true - connections[id].timestamp = now - 1000 * 60 * 60 * 1.5 + const warmupDuration = 1000 * 60 * 60 * 1.5 + connections[id].timestamp = now - warmupDuration + connections[id].startedAt = connections[id].timestamp - warmupDuration + connections[id].hit = 10 } } else { connections[id].restarts++ diff --git a/src/storage/influx.js b/src/storage/influx.js index 624a024..3f60ab3 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -4,7 +4,7 @@ const net = require('net') const config = require('../config') const socketService = require('../services/socket') const alertService = require('../services/alert') -const { connections, updateIndexes } = require('../services/connections') +const { updateIndexes } = require('../services/connections') const { parseMarket } = require('../services/catalog') require('../typedef') @@ -31,6 +31,11 @@ class InfluxStorage { */ this.pendingBars = {} + /** + * @type {{[identifier: string]: {[timestamp: number]: Bar}}} + */ + this.archivedBars = {} + /** * @type {number} */ @@ -214,7 +219,7 @@ class InfluxStorage { for (let timeframe of timeframes) { const rpDuration = timeframe * config.influxRetentionPerTimeframe - const rpDurationLitteral = getHms(rpDuration, true) + const rpDurationLitteral = getHms(rpDuration).replace(/[\s,]/g, '') const rpName = config.influxRetentionPrefix + getHms(timeframe) if (!retentionsPolicies[rpName]) { @@ -416,15 +421,8 @@ class InfluxStorage { this.pendingBars[market][tradeFlooredTime] ) { bars[market] = this.pendingBars[market][tradeFlooredTime] - //console.log(`\tuse pending bar (time ${new Date(bars[market].time).toISOString().split('T').pop().replace(/\..*/, '')})`) - } else if ( - connections[market].bar && - connections[market].bar.time === tradeFlooredTime - ) { - // trades passed in save() contains some of the last batch (trade time = last bar time) - bars[market] = this.pendingBars[market][tradeFlooredTime] = - connections[market].bar - //console.log(`\tuse connection bar (time ${new Date(bars[market].time).toISOString().split('T').pop().replace(/\..*/, '')})`) + } else if (this.archivedBars[market] && this.archivedBars[market][tradeFlooredTime]) { + bars[market] = this.pendingBars[market][tradeFlooredTime] = this.archivedBars[market][tradeFlooredTime] } else { bars[market] = this.pendingBars[market][tradeFlooredTime] = { time: tradeFlooredTime, @@ -465,22 +463,6 @@ class InfluxStorage { } } - for (const market in this.pendingBars) { - if (!connections[market]) { - continue - } - - for (const timestamp in this.pendingBars[market]) { - connections[market].bar = this.pendingBars[market][timestamp] - - if (this.pendingBars[market][timestamp].open === null) { - continue - } - - connections[market].close = this.pendingBars[market][timestamp].close - } - } - await updateIndexes(ranges, async (index, high, low, direction) => { await alertService.checkPriceCrossover(index, high, low, direction) }) @@ -495,6 +477,20 @@ class InfluxStorage { if (resampleRange.to - resampleRange.from >= 0) { await this.resample(resampleRange) } + + this.cleanArchivedBars() + } + + cleanArchivedBars() { + const limit = Date.now() - config.influxTimeframe * 100 + for (const identifier in this.archivedBars) { + for (const timestamp in this.archivedBars[identifier]) { + if (Number(timestamp) < limit) { + delete this.archivedBars[identifier][timestamp] + continue + } + } + } } /** @@ -533,13 +529,6 @@ class InfluxStorage { for (const timestamp in this.pendingBars[identifier]) { if (timestamp < now - this.influxTimeframeRetentionDuration) { - console.log( - `[storage/influx] ${new Date( - +timestamp - ).toISOString()} < ${new Date( - now - this.influxTimeframeRetentionDuration - ).toISOString()}` - ) continue } @@ -549,6 +538,12 @@ class InfluxStorage { importedRange.to = Math.max(bar.time, importedRange.to) barsToImport.push(bar) + + if (!this.archivedBars[identifier]) { + this.archivedBars[identifier] = {} + } + + this.archivedBars[identifier][timestamp] = this.pendingBars[identifier][timestamp] } } From ad669cee5f7cab407ea6d675d090e2b89c411815 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Wed, 11 Oct 2023 11:05:11 +0200 Subject: [PATCH 16/20] style: :art: lint fix --- src/config.js | 20 +++++++--- src/exchange.js | 47 +++++++++++++++-------- src/exchanges/binance.js | 2 +- src/exchanges/binance_futures.js | 8 ++-- src/exchanges/bitfinex.js | 8 ++-- src/exchanges/mexc.js | 3 +- src/exchanges/okex.js | 65 ++++++++++++++++++++++---------- src/helper.js | 9 +++-- src/server.js | 15 ++++---- src/services/alert.js | 21 ++++++++--- src/services/catalog.js | 26 ++++++++++--- src/services/connections.js | 65 ++++++++++++++++++-------------- src/storage/influx.js | 51 +++++++++++++++---------- 13 files changed, 221 insertions(+), 119 deletions(-) diff --git a/src/config.js b/src/config.js index ec0ddae..3904319 100644 --- a/src/config.js +++ b/src/config.js @@ -182,7 +182,7 @@ const defaultConfig = { alertExpiresAfter: 1000 * 60 * 60 * 24 * 7, alertEndpointExpiresAfter: 1000 * 60 * 60 * 24 * 30, indexExchangeBlacklist: [], - indexQuoteWhitelist: ['USD','USDT','BUSD','USDC'], + indexQuoteWhitelist: ['USD', 'USDT', 'BUSD', 'USDC'], // verbose debug: false @@ -298,14 +298,24 @@ if (config.whitelist && config.whitelist === 'string') { config.whitelist = [] } -if (typeof config.indexExchangeBlacklist === 'string' && config.indexExchangeBlacklist.trim().length) { - config.indexExchangeBlacklist = config.indexExchangeBlacklist.split(',').map(a => a.trim()) +if ( + typeof config.indexExchangeBlacklist === 'string' && + config.indexExchangeBlacklist.trim().length +) { + config.indexExchangeBlacklist = config.indexExchangeBlacklist + .split(',') + .map(a => a.trim()) } else if (!Array.isArray(config.indexExchangeBlacklist)) { config.indexExchangeBlacklist = [] } -if (typeof config.indexQuoteWhitelist === 'string' && config.indexQuoteWhitelist.trim().length) { - config.indexQuoteWhitelist = config.indexQuoteWhitelist.split(',').map(a => a.trim()) +if ( + typeof config.indexQuoteWhitelist === 'string' && + config.indexQuoteWhitelist.trim().length +) { + config.indexQuoteWhitelist = config.indexQuoteWhitelist + .split(',') + .map(a => a.trim()) } else if (!Array.isArray(config.indexQuoteWhitelist)) { config.indexQuoteWhitelist = [] } diff --git a/src/exchange.js b/src/exchange.js index c878c84..3741fae 100644 --- a/src/exchange.js +++ b/src/exchange.js @@ -255,7 +255,8 @@ class Exchange extends EventEmitter { if (!/ping|pong/i.test(data)) { console.debug( - `[${this.id}.createWs] sending ${data.substr(0, 64)}${data.length > 64 ? '...' : '' + `[${this.id}.createWs] sending ${data.substr(0, 64)}${ + data.length > 64 ? '...' : '' } to ${api.url}` ) } @@ -301,7 +302,11 @@ class Exchange extends EventEmitter { this.onApiCreated(api) resolve(api) } else { - reject(new Error('Failed to establish a websocket connection with exchange')) + reject( + new Error( + 'Failed to establish a websocket connection with exchange' + ) + ) } } }) @@ -311,7 +316,8 @@ class Exchange extends EventEmitter { async subscribePendingPairs(api) { console.debug( - `[${this.id}.subscribePendingPairs] subscribe to ${api._pending.length + `[${this.id}.subscribePendingPairs] subscribe to ${ + api._pending.length } pairs of api ${api.url} (${api._pending.join(', ')})` ) @@ -352,7 +358,8 @@ class Exchange extends EventEmitter { } console.debug( - `[${this.id}.unlink] disconnecting ${pair} ${skipSending ? '(skip sending)' : '' + `[${this.id}.unlink] disconnecting ${pair} ${ + skipSending ? '(skip sending)' : '' }` ) @@ -396,7 +403,7 @@ class Exchange extends EventEmitter { this.apis[i].url === url && (!this.maxConnectionsPerApi || this.apis[i]._connected.length + this.apis[i]._pending.length < - this.maxConnectionsPerApi) + this.maxConnectionsPerApi) ) { return this.apis[i] } @@ -465,7 +472,8 @@ class Exchange extends EventEmitter { api.onmessage = null console.debug( - `[${this.id}.reconnectApi] reconnect api ${api.id}${reason ? ' reason: ' + reason : '' + `[${this.id}.reconnectApi] reconnect api ${api.id}${ + reason ? ' reason: ' + reason : '' } (url: ${api.url}, _connected: ${api._connected.join( ', ' )}, _pending: ${api._pending.join(', ')})` @@ -490,7 +498,8 @@ class Exchange extends EventEmitter { pairsToReconnect ).then(() => { console.log( - `[${this.id}.reconnectApi] done reconnecting api (was ${api.id}${reason ? ' because of ' + reason : '' + `[${this.id}.reconnectApi] done reconnecting api (was ${api.id}${ + reason ? ' because of ' + reason : '' })` ) delete this.promisesOfApiReconnections[api.id] @@ -525,7 +534,9 @@ class Exchange extends EventEmitter { this.recoveryRanges.push(range) if (!recovering[this.id]) { - console.log(`[${this.id}.registerRangeForRecovery] exchange isn't recovering yet -> start recovering`) + console.log( + `[${this.id}.registerRangeForRecovery] exchange isn't recovering yet -> start recovering` + ) this.recoverNextRange() } } @@ -535,7 +546,9 @@ class Exchange extends EventEmitter { if (!this.recoveryRanges.length) { console.log(`[${this.id}] no more range to recover`) if (sequencial) { - console.log(`[${this.id}] recoverNextRange was called sequentially yet no recoveryRanges are left to recover (impossible case)`) + console.log( + `[${this.id}] recoverNextRange was called sequentially yet no recoveryRanges are left to recover (impossible case)` + ) delete recovering[this.id] } } @@ -544,14 +557,15 @@ class Exchange extends EventEmitter { const range = this.recoveryRanges.shift() - const originalRange = { + const originalRange = { from: range.from, - to: range.to, + to: range.to } const missingTime = range.to - range.from console.log( - `[${this.id}.recoverTrades] get missing trades for ${range.pair + `[${this.id}.recoverTrades] get missing trades for ${ + range.pair } (expecting ${range.missEstimate} on a ${getHms( missingTime )} blackout going from ${new Date(range.from) @@ -567,7 +581,8 @@ class Exchange extends EventEmitter { if (recoveredCount) { console.info( - `[${this.id}.recoverTrades] recovered ${recoveredCount} (expected ${range.missEstimate + `[${this.id}.recoverTrades] recovered ${recoveredCount} (expected ${ + range.missEstimate }) trades on ${this.id}:${range.pair} (${getHms( missingTime - (range.to - range.from) )} recovered out of ${getHms(missingTime)} | ${getHms( @@ -576,7 +591,8 @@ class Exchange extends EventEmitter { ) } else { console.info( - `[${this.id}.recoverTrades] 0 trade recovered on ${range.pair + `[${this.id}.recoverTrades] 0 trade recovered on ${ + range.pair } (expected ${range.missEstimate} for ${getHms( missingTime )} blackout)` @@ -829,7 +845,8 @@ class Exchange extends EventEmitter { this.failedConnections++ const delay = 1000 * this.failedConnections console.debug( - `[${this.id}] api refused connection, sleeping ${delay / 1000 + `[${this.id}] api refused connection, sleeping ${ + delay / 1000 }s before trying again` ) await sleep(delay) diff --git a/src/exchanges/binance.js b/src/exchanges/binance.js index a983946..62a2217 100644 --- a/src/exchanges/binance.js +++ b/src/exchanges/binance.js @@ -12,7 +12,7 @@ class Binance extends Exchange { this.subscriptions = {} this.endpoints = { - PRODUCTS: 'https://data-api.binance.vision/api/v3/exchangeInfo', + PRODUCTS: 'https://data-api.binance.vision/api/v3/exchangeInfo' } this.url = () => `wss://data-stream.binance.vision:9443/ws` diff --git a/src/exchanges/binance_futures.js b/src/exchanges/binance_futures.js index 5ac494b..c8bb076 100644 --- a/src/exchanges/binance_futures.js +++ b/src/exchanges/binance_futures.js @@ -162,10 +162,10 @@ class BinanceFutures extends Exchange { } /** - * - * @param {} trade - * @param {*} symbol - * @return {Trade} + * + * @param {} trade + * @param {*} symbol + * @return {Trade} */ formatTrade(trade, symbol) { return { diff --git a/src/exchanges/bitfinex.js b/src/exchanges/bitfinex.js index 44d31ab..d404cd1 100644 --- a/src/exchanges/bitfinex.js +++ b/src/exchanges/bitfinex.js @@ -63,7 +63,9 @@ class Bitfinex extends Exchange { } if (api._connected.length === 0) { - const chanId = Object.keys(this.channels[api.id]).find((chanId) => this.channels[api.id][chanId].name === 'status') + const chanId = Object.keys(this.channels[api.id]).find( + chanId => this.channels[api.id][chanId].name === 'status' + ) if (chanId) { api.send( @@ -78,7 +80,7 @@ class Bitfinex extends Exchange { } const channelsToUnsubscribe = Object.keys(this.channels[api.id]).filter( - (chanId) => this.channels[api.id][chanId].pair === pair + chanId => this.channels[api.id][chanId].pair === pair ) if (!channelsToUnsubscribe.length) { @@ -93,7 +95,7 @@ class Bitfinex extends Exchange { api.send( JSON.stringify({ event: 'unsubscribe', - chanId, + chanId }) ) delete this.channels[api.id][chanId] diff --git a/src/exchanges/mexc.js b/src/exchanges/mexc.js index e88a4ac..56f5137 100644 --- a/src/exchanges/mexc.js +++ b/src/exchanges/mexc.js @@ -12,7 +12,8 @@ class Mexc extends Exchange { this.endpoints = { PRODUCTS: [ 'https://api.mexc.com/api/v3/exchangeInfo', - 'https://contract.mexc.com/api/v1/contract/detail'] + 'https://contract.mexc.com/api/v1/contract/detail' + ] } this.url = pair => { diff --git a/src/exchanges/okex.js b/src/exchanges/okex.js index 935ce9b..3e0d280 100644 --- a/src/exchanges/okex.js +++ b/src/exchanges/okex.js @@ -36,7 +36,6 @@ class Okex extends Exchange { const type = product.instType const pair = product.instId - if (type === 'FUTURES') { // futures @@ -164,10 +163,7 @@ class Okex extends Exchange { ) }, []) - return this.emitLiquidations( - api.id, - liqs - ) + return this.emitLiquidations(api.id, liqs) } return this.emitTrades( @@ -216,7 +212,11 @@ class Okex extends Exchange { getLiquidationsUrl(range) { // after query param = before // (get the 100 trades prededing endTimestamp) - return `${this.endpoints.LIQUIDATIONS}?instId=${range.pair}&instType=SWAP&uly=${range.pair.replace('-SWAP', '')}&state=filled&after=${range.to}` + return `${this.endpoints.LIQUIDATIONS}?instId=${ + range.pair + }&instType=SWAP&uly=${range.pair.replace('-SWAP', '')}&state=filled&after=${ + range.to + }` } /** @@ -242,7 +242,11 @@ class Okex extends Exchange { const allLiquidations = [] while (true) { - console.log(`[${this.id}] fetch all ${range.pair} liquidations in`, new Date(range.from).toISOString(), new Date(range.to).toISOString()) + console.log( + `[${this.id}] fetch all ${range.pair} liquidations in`, + new Date(range.from).toISOString(), + new Date(range.to).toISOString() + ) const liquidations = await this.fetchLiquidationOrders(range) if (!liquidations || liquidations.length === 0) { @@ -251,10 +255,18 @@ class Okex extends Exchange { } console.log('received', liquidations.length, 'liquidations -> process') - console.log('first liq @ ', new Date(+liquidations[0].ts).toISOString(), new Date(+liquidations[liquidations.length - 1].ts).toISOString()) + console.log( + 'first liq @ ', + new Date(+liquidations[0].ts).toISOString(), + new Date(+liquidations[liquidations.length - 1].ts).toISOString() + ) for (const liquidation of liquidations) { if (liquidation.ts < range.from) { - console.log(`liquidation ${liquidations.indexOf(liquidation) + 1}/${liquidations.length} is outside range -> break`) + console.log( + `liquidation ${liquidations.indexOf(liquidation) + 1}/${ + liquidations.length + } is outside range -> break` + ) return allLiquidations } @@ -262,7 +274,10 @@ class Okex extends Exchange { } // new range - console.log(`[${this.id}] set new end date to last liquidation`, new Date(+liquidations[liquidations.length - 1].ts).toISOString()) + console.log( + `[${this.id}] set new end date to last liquidation`, + new Date(+liquidations[liquidations.length - 1].ts).toISOString() + ) range.to = +liquidations[liquidations.length - 1].ts } } @@ -270,19 +285,25 @@ class Okex extends Exchange { async getMissingTrades(range, totalRecovered = 0, beforeTradeId) { if (this.types[range.pair] !== 'SPOT' && !beforeTradeId) { const liquidations = await this.fetchAllLiquidationOrders({ ...range }) - console.log(`[${this.id}.recoverMissingTrades] +${liquidations.length} liquidations for ${range.pair}`) + console.log( + `[${this.id}.recoverMissingTrades] +${liquidations.length} liquidations for ${range.pair}` + ) if (liquidations.length) { - this.emitLiquidations(null, liquidations.map( - liquidation => this.formatLiquidation(liquidation, range.pair) - )) + this.emitLiquidations( + null, + liquidations.map(liquidation => + this.formatLiquidation(liquidation, range.pair) + ) + ) } } let endpoint if (beforeTradeId) { - endpoint = `https://www.okex.com/api/v5/market/history-trades?instId=${range.pair - }&limit=100${beforeTradeId ? '&after=' + beforeTradeId : ''}` + endpoint = `https://www.okex.com/api/v5/market/history-trades?instId=${ + range.pair + }&limit=100${beforeTradeId ? '&after=' + beforeTradeId : ''}` } else { endpoint = `https://www.okex.com/api/v5/market/trades?instId=${range.pair}&limit=500` } @@ -296,7 +317,11 @@ class Okex extends Exchange { const earliestTradeTime = +response.data.data[response.data.data.length - 1].ts const trades = response.data.data - .filter(trade => Number(trade.ts) > range.from && (beforeTradeId || Number(trade.ts) < range.to)) + .filter( + trade => + Number(trade.ts) > range.from && + (beforeTradeId || Number(trade.ts) < range.to) + ) .map(trade => this.formatTrade(trade)) if (trades.length) { @@ -314,7 +339,8 @@ class Okex extends Exchange { earliestTradeTime >= range.from ) { console.log( - `[${this.id}.recoverMissingTrades] +${trades.length} ${range.pair + `[${this.id}.recoverMissingTrades] +${trades.length} ${ + range.pair } ... but theres more (${getHms(remainingMissingTime)} remaining)` ) return this.waitBeforeContinueRecovery().then(() => @@ -322,7 +348,8 @@ class Okex extends Exchange { ) } else { console.log( - `[${this.id}.recoverMissingTrades] +${trades.length} ${range.pair + `[${this.id}.recoverMissingTrades] +${trades.length} ${ + range.pair } (${getHms(remainingMissingTime)} remaining)` ) } diff --git a/src/helper.js b/src/helper.js index d965652..be3665d 100644 --- a/src/helper.js +++ b/src/helper.js @@ -111,20 +111,21 @@ module.exports = { formatAmount(amount, decimals) { const negative = amount < 0 - + amount = Math.abs(amount) - + if (amount >= 1000000000) { amount = +(amount / 1000000000).toFixed(isNaN(decimals) ? 1 : decimals) + ' B' } else if (amount >= 1000000) { - amount = +(amount / 1000000).toFixed(isNaN(decimals) ? 1 : decimals) + ' M' + amount = + +(amount / 1000000).toFixed(isNaN(decimals) ? 1 : decimals) + ' M' } else if (amount >= 1000) { amount = +(amount / 1000).toFixed(isNaN(decimals) ? 1 : decimals) + ' K' } else { amount = +amount.toFixed(isNaN(decimals) ? 2 : decimals) } - + if (negative) { return '-' + amount } else { diff --git a/src/server.js b/src/server.js index 34c2d39..241ad15 100644 --- a/src/server.js +++ b/src/server.js @@ -22,7 +22,7 @@ const { restoreConnections, recovering, updateConnectionStats, - dumpConnections, + dumpConnections } = require('./services/connections') class Server extends EventEmitter { @@ -46,9 +46,7 @@ class Server extends EventEmitter { this.BANNED_IPS = [] if (config.collect) { - console.log( - `\n[server] collect is enabled`, - ) + console.log(`\n[server] collect is enabled`) console.log(`\tconnect to -> ${this.exchanges.map(a => a.id).join(', ')}`) this.handleExchangesEvents() @@ -259,9 +257,7 @@ class Server extends EventEmitter { console.error( `[${exchange.id}] api closed unexpectedly (${apiId}, ${ event.code - }, ${reason}) (was handling ${pairs.join( - ',' - )})` + }, ${reason}) (was handling ${pairs.join(',')})` ) setTimeout(() => { @@ -850,7 +846,10 @@ class Server extends EventEmitter { for (const market in alertService.alerts) { for (const range in alertService.alerts[market]) { for (const alert of alertService.alerts[market][range]) { - if (alertService.alertEndpoints[alert.endpoint] && alertService.alertEndpoints[alert.endpoint].user === user) { + if ( + alertService.alertEndpoints[alert.endpoint] && + alertService.alertEndpoints[alert.endpoint].user === user + ) { alertService.sendAlert(alert, alert.market, Date.now()) return `${alert.market} @${alert.price}` } diff --git a/src/services/alert.js b/src/services/alert.js index 5ab68c3..33ba4df 100644 --- a/src/services/alert.js +++ b/src/services/alert.js @@ -174,7 +174,11 @@ class AlertService extends EventEmitter { this.alertEndpoints[alert.endpoint].timestamp = now - console.log(`[alert/${alert.user}] create alert ${market} @${alert.price} (ajusted to ${alert.price - (priceOffset || 0)})`) + console.log( + `[alert/${alert.user}] create alert ${market} @${ + alert.price + } (ajusted to ${alert.price - (priceOffset || 0)})` + ) if (alert.message) { console.log(`\t 💬 ${alert.message}`) @@ -262,7 +266,9 @@ class AlertService extends EventEmitter { this.alerts[activeAlert.market][rangePrice].splice(index, 1) console.log( - `[alert/${activeAlert.user}] move alert on ${activeAlert.market} @${activeAlert.price} -> ${newAlert.price} (ajusted to ${ + `[alert/${activeAlert.user}] move alert on ${activeAlert.market} @${ + activeAlert.price + } -> ${newAlert.price} (ajusted to ${ activeAlert.price - (priceOffset || 0) })` ) @@ -626,12 +632,17 @@ class AlertService extends EventEmitter { continue } - const isTriggered = alert.priceCompare >= low && alert.priceCompare <= high + const isTriggered = + alert.priceCompare >= low && alert.priceCompare <= high if (isTriggered) { - console.log(`[alert/checkPriceCrossover] ${alert.price} (ajusted to ${alert.priceCompare}) ${market} ${low} <-> ${high}`) + console.log( + `[alert/checkPriceCrossover] ${alert.price} (ajusted to ${alert.priceCompare}) ${market} ${low} <-> ${high}` + ) if (debugIndexes[market]) { - console.log(`debug index ${market}: ${debugIndexes[market].join(',')}`) + console.log( + `debug index ${market}: ${debugIndexes[market].join(',')}` + ) } await this.sendAlert(alert, market, now, direction) diff --git a/src/services/catalog.js b/src/services/catalog.js index 30902e7..6945d69 100644 --- a/src/services/catalog.js +++ b/src/services/catalog.js @@ -2,10 +2,23 @@ const axios = require('axios') const fs = require('fs') const { ensureDirectoryExists } = require('../helper') -const stablecoins = ['USDT', 'USDC', 'TUSD', 'BUSD', 'USDD', 'USDK', 'USDP', 'UST'] +const stablecoins = [ + 'USDT', + 'USDC', + 'TUSD', + 'BUSD', + 'USDD', + 'USDK', + 'USDP', + 'UST' +] const currencies = ['EUR', 'USD', 'GBP', 'AUD', 'CAD', 'CHF'] -const currencyPairLookup = new RegExp(`^([A-Z0-9]{2,})[-/:]?(${currencies.join('|')})$`) -const stablecoinPairLookup = new RegExp(`^([A-Z0-9]{2,})[-/:_]?(${stablecoins.join('|')})$`) +const currencyPairLookup = new RegExp( + `^([A-Z0-9]{2,})[-/:]?(${currencies.join('|')})$` +) +const stablecoinPairLookup = new RegExp( + `^([A-Z0-9]{2,})[-/:_]?(${stablecoins.join('|')})$` +) const simplePairLookup = new RegExp(`^([A-Z0-9]{2,})[-/_]?([A-Z0-9]{3,})$`) require('../typedef') @@ -185,7 +198,10 @@ module.exports.parseMarket = function (market, noStable = true) { } else if (exchangeId === 'DERIBIT') { localSymbol = localSymbol.replace(/_(\w+)-PERPETUAL/i, '$1') } else if (exchangeId === 'BITGET') { - localSymbol = localSymbol.replace('USD_DMCBL', 'USD').replace('PERP_CMCBL', 'USDC').replace(/_.*/, '') + localSymbol = localSymbol + .replace('USD_DMCBL', 'USD') + .replace('PERP_CMCBL', 'USDC') + .replace(/_.*/, '') } else if (exchangeId === 'KUCOIN') { localSymbol = localSymbol.replace(/M$/, '') } @@ -246,6 +262,6 @@ module.exports.parseMarket = function (market, noStable = true) { pair: symbol, local: localSymbolAlpha, exchange: exchangeId, - type, + type } } diff --git a/src/services/connections.js b/src/services/connections.js index 3bc7607..d76919a 100644 --- a/src/services/connections.js +++ b/src/services/connections.js @@ -23,32 +23,35 @@ module.exports.recovering = {} */ const indexes = (module.exports.indexes = []) - ; (module.exports.registerIndexes = function () { - indexes.splice(0, indexes.length) +;(module.exports.registerIndexes = function () { + indexes.splice(0, indexes.length) - const cacheIndexes = {} + const cacheIndexes = {} - for (const market of config.pairs) { - const product = parseMarket(market) + for (const market of config.pairs) { + const product = parseMarket(market) - if (config.indexExchangeBlacklist.indexOf(product.exchange) !== -1 || config.indexQuoteWhitelist.indexOf(product.quote) === -1) { - continue - } + if ( + config.indexExchangeBlacklist.indexOf(product.exchange) !== -1 || + config.indexQuoteWhitelist.indexOf(product.quote) === -1 + ) { + continue + } - if (!cacheIndexes[product.local]) { - cacheIndexes[product.local] = { - id: product.local, - markets: [] - } + if (!cacheIndexes[product.local]) { + cacheIndexes[product.local] = { + id: product.local, + markets: [] } - - cacheIndexes[product.local].markets.push(market) } - for (const localPair in cacheIndexes) { - indexes.push(cacheIndexes[localPair]) - } - })() + cacheIndexes[product.local].markets.push(market) + } + + for (const localPair in cacheIndexes) { + indexes.push(cacheIndexes[localPair]) + } +})() /** * Read the connections file and return the content @@ -230,16 +233,16 @@ module.exports.restoreConnections = async function () { if ( !persistance[market].timestamp || (config.staleConnectionThreshold > 0 && - now - persistance[market].timestamp > - config.staleConnectionThreshold) + now - persistance[market].timestamp > config.staleConnectionThreshold) ) { console.log( - `[connections] couldn't restore ${market}'s connection because ${persistance[market].timestamp - ? `last ping is too old (${getHms( - now - persistance[market].timestamp, - true - )} ago)` - : `last ping is unknown` + `[connections] couldn't restore ${market}'s connection because ${ + persistance[market].timestamp + ? `last ping is too old (${getHms( + now - persistance[market].timestamp, + true + )} ago)` + : `last ping is unknown` }` ) // connection is to old (too much data to recover) @@ -423,13 +426,17 @@ module.exports.updateIndexes = async function (ranges, callback) { if (ranges[market]) { high += ranges[market].high low += ranges[market].low - debugIndexes[index.id].push(`${market}:rg:${ranges[market].low}-${ranges[market].high}`) + debugIndexes[index.id].push( + `${market}:rg:${ranges[market].low}-${ranges[market].high}` + ) connections[market].low = ranges[market].low connections[market].high = ranges[market].high } else { high += connections[market].close low += connections[market].close - debugIndexes[index.id].push(`${market}:co:${connections[market].close}-${connections[market].close}`) + debugIndexes[index.id].push( + `${market}:co:${connections[market].close}-${connections[market].close}` + ) connections[market].low = connections[market].close connections[market].high = connections[market].close } diff --git a/src/storage/influx.js b/src/storage/influx.js index 3f60ab3..2f062f2 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -399,7 +399,7 @@ class InfluxStorage { if (!ranges[market]) { ranges[market] = { low: trade.price, - high: trade.price, + high: trade.price } } else { ranges[market].low = Math.min(ranges[market].low, trade.price) @@ -421,8 +421,12 @@ class InfluxStorage { this.pendingBars[market][tradeFlooredTime] ) { bars[market] = this.pendingBars[market][tradeFlooredTime] - } else if (this.archivedBars[market] && this.archivedBars[market][tradeFlooredTime]) { - bars[market] = this.pendingBars[market][tradeFlooredTime] = this.archivedBars[market][tradeFlooredTime] + } else if ( + this.archivedBars[market] && + this.archivedBars[market][tradeFlooredTime] + ) { + bars[market] = this.pendingBars[market][tradeFlooredTime] = + this.archivedBars[market][tradeFlooredTime] } else { bars[market] = this.pendingBars[market][tradeFlooredTime] = { time: tradeFlooredTime, @@ -543,7 +547,8 @@ class InfluxStorage { this.archivedBars[identifier] = {} } - this.archivedBars[identifier][timestamp] = this.pendingBars[identifier][timestamp] + this.archivedBars[identifier][timestamp] = + this.pendingBars[identifier][timestamp] } } @@ -835,26 +840,32 @@ class InfluxStorage { let query = `SELECT * FROM "${config.influxDatabase}"."${config.influxRetentionPrefix}${timeframeLitteral}"."trades_${timeframeLitteral}" WHERE time >= ${from}ms AND time < ${to}ms` if (markets.length) { - query += ` AND (${markets.map((marketOrIndex) => { - if (config.influxCollectors && marketOrIndex.indexOf(':') === -1 && socketService.serverSocket) { - const collector = socketService.getNodeByMarket(marketOrIndex) - - if (collector) { - const markets = collector.markets.filter(market => { - const product = parseMarket(market) - if (product.local === marketOrIndex) { - return true + query += ` AND (${markets + .map(marketOrIndex => { + if ( + config.influxCollectors && + marketOrIndex.indexOf(':') === -1 && + socketService.serverSocket + ) { + const collector = socketService.getNodeByMarket(marketOrIndex) + + if (collector) { + const markets = collector.markets.filter(market => { + const product = parseMarket(market) + if (product.local === marketOrIndex) { + return true + } + }) + + if (markets.length) { + return markets.map(a => `market = '${a}'`).join(' OR ') } - }) - - if (markets.length) { - return markets.map(a => `market = '${a}'`).join(' OR ') } } - } - return `market = '${marketOrIndex}'` - }).join(' OR ')})` + return `market = '${marketOrIndex}'` + }) + .join(' OR ')})` } return this.influx From 169b9324a050ab3f0d6e96cc004798734239625c Mon Sep 17 00:00:00 2001 From: Tucsky Date: Wed, 11 Oct 2023 23:53:26 +0200 Subject: [PATCH 17/20] add fdusd support --- scripts/coinalize/index.js | 7 ++- src/config.js | 2 +- src/helper.js | 4 +- src/services/catalog.js | 85 ++++++++++++++++++++++++++----------- src/services/connections.js | 7 ++- src/storage/influx.js | 3 +- 6 files changed, 78 insertions(+), 30 deletions(-) diff --git a/scripts/coinalize/index.js b/scripts/coinalize/index.js index 1ad3b1b..96450f5 100644 --- a/scripts/coinalize/index.js +++ b/scripts/coinalize/index.js @@ -32,7 +32,12 @@ async function program() { continue } - const market = parseMarket(pair.market, pair.exchange === 'POLONIEX') + const market = parseMarket(pair.exchange, pair.pair, pair.exchange === 'POLONIEX') + + if (!market) { + console.warn(`${pair.market}`) + continue + } resampleRange.markets.push(market.id) diff --git a/src/config.js b/src/config.js index 3904319..c495d13 100644 --- a/src/config.js +++ b/src/config.js @@ -182,7 +182,7 @@ const defaultConfig = { alertExpiresAfter: 1000 * 60 * 60 * 24 * 7, alertEndpointExpiresAfter: 1000 * 60 * 60 * 24 * 30, indexExchangeBlacklist: [], - indexQuoteWhitelist: ['USD', 'USDT', 'BUSD', 'USDC'], + indexQuoteWhitelist: ['USD', 'USDT', 'FDUSD', 'USDC'], // verbose debug: false diff --git a/src/helper.js b/src/helper.js index be3665d..146c439 100644 --- a/src/helper.js +++ b/src/helper.js @@ -191,7 +191,7 @@ module.exports = { }, getMarkets() { return config.pairs.map(market => { - const [exchange, symbol] = market.match(/([^:]*):(.*)/).slice(1, 3) + const [exchange, pair] = market.match(/([^:]*):(.*)/).slice(1, 3) if (config.exchanges.indexOf(exchange) === -1) { console.warn(`${market} is not supported`) @@ -200,7 +200,7 @@ module.exports = { return { market, exchange, - symbol + pair } }) }, diff --git a/src/services/catalog.js b/src/services/catalog.js index 6945d69..797382a 100644 --- a/src/services/catalog.js +++ b/src/services/catalog.js @@ -2,25 +2,53 @@ const axios = require('axios') const fs = require('fs') const { ensureDirectoryExists } = require('../helper') + const stablecoins = [ 'USDT', 'USDC', 'TUSD', + 'FDUSD', 'BUSD', 'USDD', 'USDK', 'USDP', 'UST' ] -const currencies = ['EUR', 'USD', 'GBP', 'AUD', 'CAD', 'CHF'] + +const normalStablecoinLookup = /^U/ + +const normalStablecoins = stablecoins.filter(s => + normalStablecoinLookup.test(s) +) + +const reverseStablecoins = stablecoins.filter( + s => !normalStablecoinLookup.test(s) +) + +const currencies = ['EUR', 'USD', 'GBP', 'JPY', 'AUD', 'CAD', 'CHF', 'CNH'] + const currencyPairLookup = new RegExp( `^([A-Z0-9]{2,})[-/:]?(${currencies.join('|')})$` ) + +// use 2+ caracters symbol for normal stablecoins, and 3+ for reversed +// not infallible but avoids coin with symbol finishing with T or B to be labeled as TUSD or BUSD quoted markets const stablecoinPairLookup = new RegExp( - `^([A-Z0-9]{2,})[-/:_]?(${stablecoins.join('|')})$` + `^([A-Z0-9]{2,})[-/:_]?(${normalStablecoins.join( + '|' + )})$|^([A-Z0-9]{3,})[-/:_]?(${reverseStablecoins.join('|')})$` ) const simplePairLookup = new RegExp(`^([A-Z0-9]{2,})[-/_]?([A-Z0-9]{3,})$`) +const reverseStablecoinPairLookup = new RegExp( + `(\\w{3,})(${reverseStablecoins.join('|')})$`, + 'i' +) +const standardCurrencyPairLookup = new RegExp( + `(\\w{3})?(${currencies.join('|')})[a-z]?$`, + 'i' +) + require('../typedef') module.exports.saveProducts = async function (exchangeId, data) { @@ -135,17 +163,18 @@ module.exports.fetchProducts = async function (exchangeId, endpoints) { return null } -const formatStablecoin = (module.exports.formatStablecoin = function (pair) { - return pair.replace(/(\w{3})?b?usd?[a-z]?$/i, '$1USD') -}) +function stripStablePair(pair) { + return pair + .replace(reverseStablecoinPairLookup, '$1USD') + .replace(standardCurrencyPairLookup, '$1$2') +} /** * * @param {string} market * @returns {Product} */ -module.exports.parseMarket = function (market, noStable = true) { - const [exchangeId, symbol] = market.match(/([^:]*):(.*)/).slice(1, 3) +module.exports.parseMarket = function (exchangeId, symbol, noStable = true) { const id = exchangeId + ':' + symbol let type = 'spot' @@ -158,10 +187,16 @@ module.exports.parseMarket = function (market, noStable = true) { type = 'perp' } else if (exchangeId === 'HUOBI' && /_(CW|CQ|NW|NQ)$/.test(symbol)) { type = 'future' + } else if (exchangeId === 'BITMART' && !/_/.test(symbol)) { + type = 'perp' } else if (exchangeId === 'HUOBI' && /-/.test(symbol)) { type = 'perp' } else if (exchangeId === 'BYBIT' && !/-SPOT$/.test(symbol)) { - type = 'perp' + if (/.*[0-9]{2}$/.test(symbol)) { + type = 'future' + } else if (!/-SPOT$/.test(symbol)) { + type = 'perp' + } } else if ( exchangeId === 'BITMEX' || /(-|_)swap$|(-|_|:)perp/i.test(symbol) @@ -175,7 +210,10 @@ module.exports.parseMarket = function (market, noStable = true) { type = 'perp' } else if (exchangeId === 'KRAKEN' && /_/.test(symbol) && type === 'spot') { type = 'perp' - } else if (exchangeId === 'BITGET' && symbol.indexOf('_') !== -1) { + } else if ( + (exchangeId === 'BITGET' || exchangeId === 'MEXC') && + symbol.indexOf('_') !== -1 + ) { type = 'perp' } else if (exchangeId === 'KUCOIN' && symbol.indexOf('-') === -1) { type = 'perp' @@ -187,8 +225,6 @@ module.exports.parseMarket = function (market, noStable = true) { localSymbol = localSymbol.replace(/-SPOT$/, '') } else if (exchangeId === 'KRAKEN') { localSymbol = localSymbol.replace(/PI_/, '').replace(/FI_/, '') - } else if (exchangeId === 'FTX' && type === 'future') { - localSymbol = localSymbol.replace(/(\w+)-\d+$/, '$1-USD') } else if (exchangeId === 'BITFINEX') { localSymbol = localSymbol .replace(/(.*)F0:(\w+)F0/, '$1-$2') @@ -235,24 +271,25 @@ module.exports.parseMarket = function (market, noStable = true) { } } } + if (!match) { + return null + } let base let quote - if (match) { - if (match[1] === undefined && match[2] === undefined) { - base = match[3] - quote = match[4] - } else { - base = match[1] - quote = match[2] - } + if (match[1] === undefined && match[2] === undefined) { + base = match[3] + quote = match[4] || '' + } else { + base = match[1] + quote = match[2] || '' + } - if (noStable) { - localSymbolAlpha = base + formatStablecoin(quote) - } else { - localSymbolAlpha = base + quote - } + if (noStable) { + localSymbolAlpha = stripStablePair(base + quote) + } else { + localSymbolAlpha = base + quote } return { diff --git a/src/services/connections.js b/src/services/connections.js index d76919a..96a9c50 100644 --- a/src/services/connections.js +++ b/src/services/connections.js @@ -29,7 +29,12 @@ const indexes = (module.exports.indexes = []) const cacheIndexes = {} for (const market of config.pairs) { - const product = parseMarket(market) + const [exchange, pair] = market.match(/([^:]*):(.*)/).slice(1, 3) + const product = parseMarket(exchange, pair) + + if (!product) { + continue + } if ( config.indexExchangeBlacklist.indexOf(product.exchange) !== -1 || diff --git a/src/storage/influx.js b/src/storage/influx.js index 2f062f2..2515805 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -851,7 +851,8 @@ class InfluxStorage { if (collector) { const markets = collector.markets.filter(market => { - const product = parseMarket(market) + const [exchange, pair] = market.match(/([^:]*):(.*)/).slice(1, 3) + const product = parseMarket(exchange, pair) if (product.local === marketOrIndex) { return true } From 025543781e990bdcdd5b2b3d053a05a62f6a4813 Mon Sep 17 00:00:00 2001 From: Tucsky Date: Thu, 12 Oct 2023 02:54:06 +0200 Subject: [PATCH 18/20] fix alert ranges check --- src/services/alert.js | 2 +- src/services/connections.js | 7 ++----- src/storage/influx.js | 4 +++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/services/alert.js b/src/services/alert.js index 33ba4df..e074223 100644 --- a/src/services/alert.js +++ b/src/services/alert.js @@ -628,7 +628,7 @@ class AlertService extends EventEmitter { for (let i = 0; i < this.alerts[market][rangePrice].length; i++) { const alert = this.alerts[market][rangePrice][i] - if (now - alert.timestamp < config.influxTimeframe) { + if (now - alert.timestamp < config.backupInterval) { continue } diff --git a/src/services/connections.js b/src/services/connections.js index 96a9c50..d914422 100644 --- a/src/services/connections.js +++ b/src/services/connections.js @@ -423,7 +423,7 @@ module.exports.updateIndexes = async function (ranges, callback) { if ( !connections[market] || !connections[market].apiId || - !connections[market].close + (!connections[market].close && !ranges[market]) ) { continue } @@ -434,16 +434,13 @@ module.exports.updateIndexes = async function (ranges, callback) { debugIndexes[index.id].push( `${market}:rg:${ranges[market].low}-${ranges[market].high}` ) - connections[market].low = ranges[market].low - connections[market].high = ranges[market].high + connections[market].close = ranges[market].close } else { high += connections[market].close low += connections[market].close debugIndexes[index.id].push( `${market}:co:${connections[market].close}-${connections[market].close}` ) - connections[market].low = connections[market].close - connections[market].high = connections[market].close } close += connections[market].close diff --git a/src/storage/influx.js b/src/storage/influx.js index 2515805..7256782 100644 --- a/src/storage/influx.js +++ b/src/storage/influx.js @@ -399,11 +399,13 @@ class InfluxStorage { if (!ranges[market]) { ranges[market] = { low: trade.price, - high: trade.price + high: trade.price, + close: trade.price } } else { ranges[market].low = Math.min(ranges[market].low, trade.price) ranges[market].high = Math.max(ranges[market].high, trade.price) + ranges[market].close = trade.price } } From 73cffc359a82a3386d6d242d64f8c5ad9c763d35 Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Sat, 14 Oct 2023 02:17:05 +0200 Subject: [PATCH 19/20] chore: set version to 1.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 21dde82..7b0fb7a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aggr-server", - "version": "1.0.2", + "version": "1.1.0", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", From cb8dda293540492240df898ce0b95b2a9253941b Mon Sep 17 00:00:00 2001 From: Axel De Acetis Date: Sat, 14 Oct 2023 02:25:45 +0200 Subject: [PATCH 20/20] feat: add bitmart Co-authored-by: Lars van der Zande --- package-lock.json | 2 +- package.json | 2 +- src/exchanges/bitmart.js | 233 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 src/exchanges/bitmart.js diff --git a/package-lock.json b/package-lock.json index 4561174..22b177d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "express-rate-limit": "^5.1.1", "influx": "^5.9.3", "jsoning": "^0.13.23", - "pako": "^1.0.6", + "pako": "^1.0.11", "pm2": "^5.3.0", "tx2": "^1.0.5", "uuid": "^9.0.0", diff --git a/package.json b/package.json index 7b0fb7a..d2af006 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "express-rate-limit": "^5.1.1", "influx": "^5.9.3", "jsoning": "^0.13.23", - "pako": "^1.0.6", + "pako": "^1.0.11", "pm2": "^5.3.0", "tx2": "^1.0.5", "uuid": "^9.0.0", diff --git a/src/exchanges/bitmart.js b/src/exchanges/bitmart.js new file mode 100644 index 0000000..ee5a29d --- /dev/null +++ b/src/exchanges/bitmart.js @@ -0,0 +1,233 @@ +const Exchange = require('../exchange') +const { sleep, getHms } = require('../helper') +const axios = require('axios') + +const { inflateRaw } = require('pako') + +/** + * Bitmart class representing the Bitmart exchange. + * @extends Exchange + */ +class Bitmart extends Exchange { + constructor() { + super() + + /** + * @type {string} + */ + this.id = 'BITMART' + + /** + * @type {object} + */ + this.subscriptions = {} + + /** + * @type {object} + */ + this.endpoints = { + /** + * URLs for product details on the Bitmart spot and contract markets. + * @type {string[]} + */ + PRODUCTS: [ + 'https://api-cloud.bitmart.com/spot/v1/symbols/details', + 'https://api-cloud.bitmart.com/contract/public/details' + ], + SPOT: { + RECENT_TRADES: 'https://api-cloud.bitmart.com/spot/quotation/v3/trades' + }, + FUTURES: { + RECENT_TRADES: '' + } + } + } + + async getUrl(pair) { + if (this.specs[pair]) { + return 'wss://openapi-ws.bitmart.com/api?protocol=1.1' + } + return 'wss://ws-manager-compress.bitmart.com/api?protocol=1.1' + } + + validateProducts(data) { + if (!data.specs) { + return false + } + + return true + } + + formatProducts(responses) { + const products = [] + const specs = {} + const types = {} + + for (const response of responses) { + for (const product of response.data.symbols) { + products.push(product.symbol) + + if (product.contract_size) { + specs[product.symbol] = +product.contract_size + } + } + } + + return { products, specs, types } + } + + /** + * Sub + * @param {WebSocket} api + * @param {string} pair + */ + async subscribe(api, pair) { + if (!(await super.subscribe(api, pair))) { + return + } + + const isContract = !!this.specs[pair] + + const typeImpl = isContract + ? { + prefix: 'futures', + arg: 'action' + } + : { + prefix: 'spot', + arg: 'op' + } + + api.send( + JSON.stringify({ + [typeImpl.arg]: `subscribe`, + args: [`${typeImpl.prefix}/trade:${pair}`] + }) + ) + + return true + } + + /** + * Sub + * @param {WebSocket} api + * @param {string} pair + */ + async unsubscribe(api, pair) { + if (!(await super.unsubscribe(api, pair))) { + return + } + + const isContract = !!this.specs[pair] + + const typeImpl = isContract + ? { + prefix: 'futures', + arg: 'action' + } + : { + prefix: 'spot', + arg: 'op' + } + + api.send( + JSON.stringify({ + [typeImpl.arg]: `unsubscribe`, + args: [`${typeImpl.prefix}/trade:${pair}`] + }) + ) + + return true + } + + formatTrade(trade) { + if (this.specs[trade.symbol]) { + return { + exchange: this.id, + pair: trade.symbol, + timestamp: trade.create_time_mill, + price: +trade.deal_price, + size: + (trade.deal_vol * this.specs[trade.symbol]) / + (this.specs[trade.symbol] > 1 ? trade.deal_price : 1), + side: trade.way < 4 ? 'buy' : 'sell' + } + } else { + return { + exchange: this.id, + pair: trade.symbol, + timestamp: trade.s_t * 1000, + price: +trade.price, + size: +trade.size, + side: trade.side + } + } + } + + onMessage(message, api) { + let data = message.data + + if (typeof data !== 'string') { + data = inflateRaw(message.data, { to: 'string' }) + } + + const json = JSON.parse(data) + + if (!json || !json.data) { + throw Error(`${this.id}: Can't parse json data from messageEvent`) + } + const trades = json.data.map(trade => this.formatTrade(trade)) + this.emitTrades( + api.id, + json.data.map(trade => this.formatTrade(trade)) + ) + } + + getMissingTrades(range, totalRecovered = 0) { + // limit to 50 trades and no way to go back further + let endpoint = + this.endpoints.SPOT.RECENT_TRADES + `?symbol=${range.pair.toUpperCase()}` + + const mapResponseToTradeFormat = trade => { + return { + exchange: this.id, + pair: trade[0], + timestamp: trade[1], + price: trade[2], + size: trade[3], + side: trade[4] + } + } + return axios + .get(endpoint) + .then(response => { + if (response.data.length) { + const trades = response.data.map(trade => + mapResponseToTradeFormat(trade) + ) + + this.emitTrades(null, trades) + + totalRecovered += trades.length + + console.log( + `[${this.id}.recoverMissingTrades] +${trades.length} ${ + range.pair + } (${getHms(remainingMissingTime)} remaining)` + ) + } + + return totalRecovered + }) + .catch(err => { + console.error( + `[${this.id}] failed to get missing trades on ${range.pair}`, + err.message + ) + + return totalRecovered + }) + } +} + +module.exports = Bitmart