From 6c4371a1b4f3fef7cebdf57912c28906f8b74dd2 Mon Sep 17 00:00:00 2001 From: Tobiah Date: Sat, 30 Jan 2021 19:39:32 -0600 Subject: [PATCH] feat: cache guilds & common large queries to flatfiles hydrate caches on startup if not present rehydrate caches hourly, queries every 10 minutes --- .gitignore | 3 +- package-lock.json | 88 ++++++++++----- package.json | 2 + src/WorldStateCache.js | 4 +- src/embeds/EarthCycleEmbed.js | 2 +- src/notifications/Broadcaster.js | 29 +++-- src/notifications/Notifier.js | 109 +++++++++---------- src/notifications/Worker.js | 61 ++++++++++- src/resources/Fetcher.js | 12 +- src/resources/cachedEvents.json | 1 + src/resources/missionTypes.json | 4 + src/settings/DatabaseQueries/PingsQueries.js | 29 ++--- 12 files changed, 219 insertions(+), 125 deletions(-) create mode 100644 src/resources/cachedEvents.json diff --git a/.gitignore b/.gitignore index 376e8b0e3..5f862ffb3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ jspm_packages #ignore prod pm2 config genesis.json worker.json +worker.dev.json .idea/ mypm2.json @@ -50,4 +51,4 @@ mypm2.json commands.json # Docker -docker-compose.yaml +docker-compose.yaml \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 9d507866b..eb6406b20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -573,8 +573,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -608,7 +607,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -757,8 +755,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "confusing-browser-globals": { "version": "1.0.9", @@ -782,6 +779,14 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cron": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/cron/-/cron-1.8.2.tgz", + "integrity": "sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==", + "requires": { + "moment-timezone": "^0.5.x" + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -951,7 +956,7 @@ } }, "erlpack": { - "version": "github:discordapp/erlpack#e27db8f82892bdb9b28a0547cc394d68b5d2242d", + "version": "github:discordapp/erlpack#c514d36ec81a7a61ef90b75df261025ab046574d", "from": "github:discordapp/erlpack", "requires": { "bindings": "^1.5.0", @@ -1341,6 +1346,34 @@ "dev": true, "requires": { "flat-cache": "^2.0.1" + }, + "dependencies": { + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + } + }, + "flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } } }, "file-uri-to-path": { @@ -1358,21 +1391,18 @@ } }, "flat-cache": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", - "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", - "dev": true, + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", "requires": { - "flatted": "^2.0.0", - "rimraf": "2.6.3", - "write": "1.0.3" + "flatted": "^3.1.0", + "rimraf": "^3.0.2" } }, "flatted": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", - "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", - "dev": true + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz", + "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==" }, "forever-agent": { "version": "0.6.1", @@ -1392,8 +1422,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "function-bind": { "version": "1.1.1", @@ -1444,7 +1473,6 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1621,7 +1649,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -2039,7 +2066,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -2062,6 +2088,14 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, + "moment-timezone": { + "version": "0.5.32", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.32.tgz", + "integrity": "sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2336,8 +2370,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-key": { "version": "2.0.1", @@ -2556,10 +2589,9 @@ } }, "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "dev": true, + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "requires": { "glob": "^7.1.3" } diff --git a/package.json b/package.json index e5b67085e..8eb9785a5 100644 --- a/package.json +++ b/package.json @@ -48,10 +48,12 @@ "bufferutil": "^4.0.3", "byte-size": "^7.0.0", "colors": "^1.4.0", + "cron": "^1.8.2", "decache": "^4.6.0", "discord-giveaways": "^3.2.3", "discord.js": "^12.5.1", "erlpack": "github:discordapp/erlpack", + "flat-cache": "^3.0.4", "json-query": "^2.2.2", "moment": "^2.29.1", "ms": "^2.1.3", diff --git a/src/WorldStateCache.js b/src/WorldStateCache.js index 444f5cd3e..fb63210d6 100644 --- a/src/WorldStateCache.js +++ b/src/WorldStateCache.js @@ -3,6 +3,7 @@ const EventEmitter = require('events'); const { apiBase } = require('./CommonFunctions.js'); const fetch = require('./resources/Fetcher'); + const logger = require('./Logger'); const worldStateURLs = { @@ -22,7 +23,6 @@ class WorldStateCache extends EventEmitter { this.updating = null; this.platform = platform; this.updateInterval = setInterval(() => this.update(), timeout); - this.logger = logger; this.update(); } @@ -42,7 +42,7 @@ class WorldStateCache extends EventEmitter { return this.currentData; } catch (err) { this.updating = undefined; - this.logger.error(err); + logger.error(err); } return this.updating; } diff --git a/src/embeds/EarthCycleEmbed.js b/src/embeds/EarthCycleEmbed.js index f8253fcb2..b2b479ad2 100644 --- a/src/embeds/EarthCycleEmbed.js +++ b/src/embeds/EarthCycleEmbed.js @@ -18,7 +18,7 @@ class EarthCycleEmbed extends BaseEmbed { constructor(bot, state) { super(); - this.title = `Worldstate - ${state.isCetus ? 'Plains of Eidolon' : 'Earth'} Cycle - ${state.isDay ? 'Day' : 'Night'}time`; + this.title = `Worldstate - ${state.isCetus ? 'PoE' : 'Earth'} - ${state.isDay ? 'Day' : 'Night'}`; this.color = state.isDay ? 0xB64624 : 0x000066; this.thumbnail = { url: state.isCetus ? ostron : earth, diff --git a/src/notifications/Broadcaster.js b/src/notifications/Broadcaster.js index 22296754e..a32ce4a86 100644 --- a/src/notifications/Broadcaster.js +++ b/src/notifications/Broadcaster.js @@ -2,6 +2,7 @@ const bs = require('byte-size'); const logger = require('../Logger'); +const cachedEvents = require('../resources/cachedEvents'); function byteFmt() { return `${this.value}${this.unit}`; @@ -37,12 +38,13 @@ const clean = (channelId, index) => { */ class Broadcaster { constructor({ - client = undefined, settings = undefined, messageManager = undefined, + client = undefined, settings = undefined, messageManager = undefined, workerCache = undefined, }) { this.client = client; this.settings = settings; this.webhook = messageManager.webhook; this.shards = process.env.SHARDS; + this.workerCache = workerCache; } wrap(embed, ctx) { @@ -72,22 +74,31 @@ class Broadcaster { * @returns {Array.} values for successes */ async broadcast(embed, platform, type, items = []) { + logger.silly(`broadcasting ${type}`); delete embed.bot; - const guilds = await this.settings.getGuilds(); - const channels = await this.settings.getAgnosticNotifications(type, platform, items); - for (const result of channels) { - const index = channels.indexOf(result); - const ctx = await this.settings.getCommandContext(result.channelId); + const guilds = this.workerCache.getKey('guilds'); + + const channels = cachedEvents.includes(type) + ? this.workerCache.getKey(`${type}:${platform}`) + : await this.settings.getAgnosticNotifications(type, platform, items); + if (!channels.length) { + logger.silly(`No channels on ${platform} tracking ${type}... continuing`); + return; + } + for (const channelId of channels) { + if (typeof channelId === 'undefined' || !channelId.length) continue; + const index = channels.indexOf(channelId); + const ctx = await this.settings.getCommandContext(channelId); // localeCompare should return 0 if equal, so non-zero's will be truthy if (embed.locale && ctx.language.localeCompare(embed.locale)) { - clean(result.channelId, index); + clean(channelId, index); continue; } const glist = Object.entries(guilds) - .filter(([, g]) => g.channels && g.channels.includes(result.channelId))[0]; + .filter(([, g]) => g.channels && g.channels.includes(channelId))[0]; const guild = glist && glist.length ? glist[1] : null; if (!guild) continue; @@ -99,7 +110,7 @@ class Broadcaster { } else { await this.webhook(ctx, { text: prepend, embed }); } - clean(result.channelId, index); + clean(channelId, index); } catch (e) { logger.error(e); } diff --git a/src/notifications/Notifier.js b/src/notifications/Notifier.js index ad0ee5b19..1785f7476 100644 --- a/src/notifications/Notifier.js +++ b/src/notifications/Notifier.js @@ -119,6 +119,7 @@ function buildNotifiableData(newData, platform, notified) { && !notified.includes(`arbitration:${new Date(newData.arbitration.expiry).getTime()}`) ? newData.arbitration : undefined, + outposts: newData.sentientOutposts.active && !notified.includes(newData.sentientOutposts.id), }; const ostron = newData.syndicateMissions.filter(mission => mission.syndicate === 'Ostrons')[0]; @@ -143,22 +144,10 @@ function buildNotifiableData(newData, platform, notified) { /** * Notifier for alerts, invasions, etc. - * TODO: remove dependence on 'bot', use something like https://github.com/spec-tacles/rest.js - * to leverage direct api routing/calls with ratelimit support - * use this in place of bot calls to queue up role changes, - * and separate the notifications from the rest of the bot functionality */ class Notifier { - /** - * * Set up essential notifier dependencies - * * Get rid of all external pull-ins - * rewrite to not use a bot client, but a direct api router - * * Instantiate our own logger - * * Instantiate our own db connection - * @param {Genesis} bot instance of the bot.... this needs to be refactored/removed - */ constructor({ - settings, client, messageManager, worldStates, timeout, + settings, client, messageManager, worldStates, timeout, workerCache, }) { this.settings = settings; this.client = client; @@ -167,6 +156,7 @@ class Notifier { client, settings: this.settings, messageManager, + workerCache, }); logger.info('[N] Ready'); @@ -178,15 +168,14 @@ class Notifier { }); this.updating = false; - refreshRate = timeout; + this.updating = []; } /** Start the notifier */ async start() { Object.entries(this.worldStates).forEach(([, ws]) => { ws.on('newData', async (platform, newData) => { - logger.silly(`[N] Processing new data for ${platform}`); await this.onNewData(platform, newData); }); }); @@ -200,8 +189,7 @@ class Notifier { async onNewData(platform, newData) { // don't wait for the previous to finish, this creates a giant backup, // adding 4 new entries every few seconds - if (this.updating) return; - // await this.updating; + if (this.updating.includes(platform)) return; beats[platform].currCycleStart = Date.now(); if (!(newData && newData.timestamp)) return; @@ -209,11 +197,12 @@ class Notifier { const notifiedIds = await this.settings.getNotifiedIds(platform); // Set up data to notify - this.updating = this.sendNew(platform, newData, notifiedIds, + this.updating.push(platform); + + await this.sendNew(platform, newData, notifiedIds, buildNotifiableData(newData, platform, notifiedIds)); - await this.updating; - this.updating = undefined; + this.updating.splice(this.updating.indexOf(platform), 1); } async sendNew(platform, rawData, notifiedIds, { @@ -222,34 +211,41 @@ class Notifier { cetusCycle, earthCycle, vallisCycle, tweets, nightwave, cetusCycleChange, earthCycleChange, vallisCycleChange, featuredDeals, streams, popularDeals, primeAccess, updates, conclave, - cambionCycle, cambionCycleChange, + cambionCycle, cambionCycleChange, outposts, }) { // Send all notifications const cycleIds = []; try { - logger.silly('[N] sending new data...'); - await this.sendAcolytes(acolytes, platform); + logger.silly(`[N] sending new data on ${platform}...`); + + this.sendAcolytes(acolytes, platform); + if (baro) { - await this.sendBaro(baro, platform); + this.sendBaro(baro, platform); } - // if (conclave && conclave.length > 0) { - // await this.sendConclaveDailies(conclave, platform); - // await this.sendConclaveWeeklies(conclave, platform); - // } - // if (tweets && tweets.length > 0) { - // await this.sendTweets(tweets, platform); - // } - // await this.sendDarvo(dailyDeals, platform); - await this.sendEvent(events, platform); - // await this.sendFeaturedDeals(featuredDeals, platform); - await this.sendFissures(fissures, platform); - // await this.sendNews(news, platform); - // await this.sendStreams(streams, platform); - // await this.sendPopularDeals(popularDeals, platform); - // await this.sendPrimeAccess(primeAccess, platform); - // await this.sendInvasions(invasions, platform); - await this.sendSortie(sortie, platform); - await this.sendSyndicates(syndicateM, platform); + if (conclave && conclave.length > 0) { + this.sendConclaveDailies(conclave, platform); + this.sendConclaveWeeklies(conclave, platform); + } + if (tweets && tweets.length > 0) { + this.sendTweets(tweets, platform); + } + this.sendDarvo(dailyDeals, platform); + this.sendEvent(events, platform); + this.sendFeaturedDeals(featuredDeals, platform); + //this.sendFissures(fissures, platform); + this.sendNews(news, platform); + this.sendStreams(streams, platform); + this.sendPopularDeals(popularDeals, platform); + this.sendPrimeAccess(primeAccess, platform); + this.sendInvasions(invasions, platform); + this.sendSortie(sortie, platform); + this.sendSyndicates(syndicateM, platform); + this.sendUpdates(updates, platform); + this.sendAlerts(alerts, platform); + this.sendSentientOutposts(outposts, platform); + this.sendNightwave(nightwave, platform); + this.sendArbitration(arbitration, platform); cycleIds.push( await this.sendCetusCycle(cetusCycle, platform, cetusCycleChange, notifiedIds), ); @@ -262,20 +258,13 @@ class Notifier { cycleIds.push( await this.sendCambionCycle(cambionCycle, platform, cambionCycleChange, notifiedIds), ); - await this.sendUpdates(updates, platform); - await this.sendAlerts(alerts, platform); - cycleIds.push( - await this.sendSentientOutposts(rawData.sentientOutposts, platform, notifiedIds), - ); - await this.sendNightwave(nightwave, platform); - await this.sendArbitration(arbitration, platform); } catch (e) { logger.error(e); } finally { beats[platform].lastUpdate = Date.now(); } - const alreadyNotified = []; - alreadyNotified.push( + + const alreadyNotified = [ ...rawData.persistentEnemies.map(a => a.pid), ...cycleIds, `${rawData.voidTrader.id}${rawData.voidTrader.active ? '1' : '0'}`, @@ -294,10 +283,13 @@ class Notifier { ? `arbitration:${new Date(rawData.arbitration.expiry).getTime()}` : 'arbitration:0', ...(rawData.twitter ? rawData.twitter.map(t => t.uniqueId) : []), - ...(rawData.nightwave.active ? rawData.nightwave.activeChallenges.map(c => c.id) : []), - ); + ...(rawData.nightwave && rawData.nightwave.active + ? rawData.nightwave.activeChallenges.map(c => c.id) + : []), + rawData.sentientOutposts.id, + ]; - return this.settings.setNotifiedIds(platform, alreadyNotified); + await this.settings.setNotifiedIds(platform, alreadyNotified); } async sendAcolytes(newAcolytes, platform) { @@ -434,7 +426,7 @@ class Notifier { for (const [locale, i18n] of Object.entries(i18ns)) { const embed = new embeds.Fissure({ logger }, [fissure], platform, i18n); embed.locale = locale; - const id = `fissures.t${fissure.tierNum}.${fissure.missionType.toLowerCase()}`; + const id = `fissures.t${fissure.tierNum}.${fissure.missionType.toLowerCase().replace(/\s/g, '')}`; await this.broadcaster.broadcast(embed, platform, id); } } @@ -565,15 +557,14 @@ class Notifier { return type; } - async sendSentientOutposts(outpost, platform, notifiedIds) { - if (outpost.active && !notifiedIds.includes(outpost.id)) { - for (const [locale, i18n] of Object.entries(i18ns)) { + async sendSentientOutposts(outpost, platform) { + for (const [locale, i18n] of Object.entries(i18ns)) { + if (outpost.mission) { const embed = new embeds.Outposts({ logger }, outpost, platform, i18n); embed.locale = locale; await this.broadcaster.broadcast(embed, platform, 'outposts'); } } - return outpost.id; } } diff --git a/src/notifications/Worker.js b/src/notifications/Worker.js index b02dbddff..a96c6762b 100644 --- a/src/notifications/Worker.js +++ b/src/notifications/Worker.js @@ -1,5 +1,8 @@ 'use strict'; +const flatCache = require('flat-cache'); +const Job = require('cron').CronJob; + const Notifier = require('./Notifier'); const FeedsNotifier = require('./FeedsNotifier'); const TwitchNotifier = require('./TwitchNotifier'); @@ -10,6 +13,9 @@ const Rest = require('../tools/RESTWrapper'); const Database = require('../settings/Database'); const { logger } = require('./NotifierUtils'); +const cachedEvents = require('../resources/cachedEvents'); + +const activePlatforms = (process.env.PLATFORMS || 'pc').split(','); const rest = new Rest(); const db = new Database(); @@ -18,6 +24,50 @@ const deps = {}; let timeout; +const hydrateGuilds = async () => { + if (!deps.workerCache) return; + const guilds = await db.getGuilds(); + if (guilds) { + deps.workerCache.setKey('guilds', guilds); + deps.workerCache.save(true); + } +}; + +const hydrateQueries = async () => { + if (!deps.workerCache) return; + for (const cachedEvent of cachedEvents) { + for (const platform of activePlatforms) { + deps.workerCache.setKey(`${cachedEvent}:${platform}`, + await db.getAgnosticNotifications(cachedEvent, platform)); + } + } + deps.workerCache.save(true); +}; + +const initCache = async () => { + deps.workerCache = flatCache.load('genesis:worker'); + + // generate guild cache data if not present + const currentGuilds = deps.workerCache.getKey('guilds'); + if (!currentGuilds) { + await hydrateGuilds(); + } + + let hydrateEvents = false; + for (const cachedEvent of cachedEvents) { + for (const platform of activePlatforms) { + if (!deps.workerCache.getKey(`${cachedEvent}:${platform}`)) { + hydrateEvents = true; + } + } + } + if (hydrateEvents) await hydrateQueries(); + + // refresh guild cache every hour... it's a heavy process, we don't want to do it much + deps.guildHydration = new Job('0 0 * * * *', hydrateGuilds); + deps.queryHydration = new Job('0 */10 * * * *', hydrateQueries); +}; + class Worker { constructor() { /** @@ -27,9 +77,9 @@ class Worker { this.worldStates = {}; timeout = process.env.WORLDSTATE_TIMEOUT || 60000; - ['pc', 'ps4', 'xb1', 'swi'] + activePlatforms .forEach((platform) => { - this.worldStates[platform] = new WorldStateCache(platform, timeout, this.logger); + this.worldStates[platform] = new WorldStateCache(platform, timeout); }); } @@ -46,6 +96,7 @@ class Worker { await rest.init(); await deps.settings.init(); + await initCache(); this.messageManager = new MessageManager(deps); deps.messageManager = this.messageManager; @@ -56,16 +107,14 @@ class Worker { this.feedNotifier.start(); this.twitchNotifier.start(); - await this.notifier.start(); - const body = { + rest.controlMessage({ embeds: [{ description: 'Worker ready!', color: 0x2B90EC, }], - }; - rest.controlMessage(body); + }); } catch (e) { logger.error(e); } diff --git a/src/resources/Fetcher.js b/src/resources/Fetcher.js index 48d42dfcc..a1b5c22df 100644 --- a/src/resources/Fetcher.js +++ b/src/resources/Fetcher.js @@ -7,8 +7,8 @@ const logger = require('../Logger'); const retryCodes = [429].concat((process.env.JSON_CACHE_RETRY_CODES || '').split(',').map(code => parseInt(code.trim(), 10))); const redirectCodes = [302, 301].concat((process.env.JSON_CACHE_REDIRECT_CODES || '').split(',').map(code => parseInt(code.trim(), 10))); -const fetch = (url, { promiseLib = Promise, maxRetry = 10, headers } = -{ promiseLib: Promise, maxRetry: 10, headers: {} }) => { +const fetch = (url, { maxRetry = 10, headers } = +{ maxRetry: 10, headers: {} }) => { const protocol = url.startsWith('https') ? https : http; // eslint-disable-next-line new-cap return new Promise((resolve) => { @@ -18,16 +18,16 @@ const fetch = (url, { promiseLib = Promise, maxRetry = 10, headers } = if (response.statusCode < 200 || response.statusCode > 299) { if (redirectCodes.includes(response.statusCode)) { setTimeout(() => { - fetch(response.headers.location, { promiseLib, maxRetry, headers }) - .then(resolve) + fetch(response.headers.location, { maxRetry, headers }) + .then(d => resolve(d)) .catch(logger.error); }, 1000); } else if ((response.statusCode > 499 || retryCodes.includes(response.statusCode)) && maxRetry > 0) { maxRetry -= 1; // eslint-disable-line no-param-reassign setTimeout(() => { - fetch(url, { promiseLib, maxRetry }) - .then(resolve) + fetch(url, { maxRetry, headers }) + .then(d => resolve(d)) .catch(logger.error); }, 1000); } else { diff --git a/src/resources/cachedEvents.json b/src/resources/cachedEvents.json new file mode 100644 index 000000000..8c592be84 --- /dev/null +++ b/src/resources/cachedEvents.json @@ -0,0 +1 @@ +["cetus.day", "cetus.night", "deimos.fass", "deimos.vome", "solaris.warm", "solaris.cold"] \ No newline at end of file diff --git a/src/resources/missionTypes.json b/src/resources/missionTypes.json index 0d9333a9a..83c109b88 100644 --- a/src/resources/missionTypes.json +++ b/src/resources/missionTypes.json @@ -1,4 +1,8 @@ { + "assault": { + "arbit": false, + "fissure": true + }, "excavation": { "arbi": true, "fissure": true diff --git a/src/settings/DatabaseQueries/PingsQueries.js b/src/settings/DatabaseQueries/PingsQueries.js index 5bf3dc189..def1b0756 100644 --- a/src/settings/DatabaseQueries/PingsQueries.js +++ b/src/settings/DatabaseQueries/PingsQueries.js @@ -5,6 +5,7 @@ const SQL = require('sql-template-strings'); class PingsQueries { constructor(db) { this.db = db; + this.scope = (process.env.SCOPE || 'worker').toLowerCase(); } /** @@ -144,22 +145,24 @@ class PingsQueries { FROM type_notifications` .append(items && items.length ? SQL` INNER JOIN item_notifications ON type_notifications.channel_id = item_notifications.channel_id` : SQL``) - .append(SQL` INNER JOIN channels ON channels.id = type_notifications.channel_id`) - .append(SQL` INNER JOIN settings as s1 ON channels.id = s1.channel_id`) - .append(SQL` INNER JOIN settings as ws1 ON channels.id = ws1.channel_id`) - .append(SQL` INNER JOIN settings as ws2 ON channels.id = ws2.channel_id`) - .append(SQL` INNER JOIN settings as ws3 ON channels.id = ws1.channel_id`) - .append(SQL` INNER JOIN settings as ws4 ON channels.id = ws2.channel_id`) .append(SQL` - WHERE type_notifications.type = ${String(type)} - AND s1.setting = "platform" AND (s1.val = ${platform || 'pc'} OR s1.val IS NULL) - and ws1.setting = "webhookToken" AND ws1.val IS NOT NULL - and ws2.setting = "webhookId" AND ws2.val IS NOT NULL - and ws3.setting = "webhookAvatar" AND ws3.val IS NOT NULL - and ws4.setting = "webhookName" AND ws4.val IS NOT NULL `) + INNER JOIN channels ON channels.id = type_notifications.channel_id + INNER JOIN settings as s1 ON channels.id = s1.channel_id + AND s1.setting = "platform" AND (s1.val = ${platform || 'pc'} OR s1.val IS NULL) + INNER JOIN settings as ws1 ON channels.id = ws1.channel_id + AND ws1.setting = "webhookToken" AND ws1.val IS NOT NULL + INNER JOIN settings as ws2 ON channels.id = ws2.channel_id + AND ws2.setting = "webhookId" AND ws2.val IS NOT NULL + INNER JOIN settings as ws3 ON channels.id = ws3.channel_id + AND ws3.setting = "webhookAvatar" AND ws3.val IS NOT NULL + INNER JOIN settings as ws4 ON channels.id = ws4.channel_id + AND ws4.setting = "webhookName" AND ws4.val IS NOT NULL `) + .append(SQL` WHERE type_notifications.type = ${String(type)} `) .append(items && items.length ? SQL` AND item_notifications.item IN (${items}) AND item_notifications.channel_id = settings.channel_id;` : SQL`;`); - return (await this.query(query))[0]; + return (await this.query(query))[0] + .map(o => o.channelId) + .filter(o => o); } catch (e) { this.logger.error(e); return [];