diff --git a/subm/app.yaml b/subm/app.yaml index ba9a882..5baa7d8 100644 --- a/subm/app.yaml +++ b/subm/app.yaml @@ -31,7 +31,10 @@ includes: # # Discord # DISCORD_API_TOKEN: sekretsekret # DISCORD_GUILD_ID: 585... -# # DISCORD_USER_ID: 358096357862408195 +# # validator-1-bld +# CHANNEL_ID: 946... +# # mod1bld +# REVIEWER_ROLE_ID: 946... # DISCORD_CLIENT_ID: 874... # DISCORD_CLIENT_SECRET: sekretsekret diff --git a/subm/package.json b/subm/package.json index 5fae8b5..cbbf2c7 100644 --- a/subm/package.json +++ b/subm/package.json @@ -9,6 +9,7 @@ "lint:eslint": "eslint '**/*.js'", "lint:types": "tsc -p jsconfig.json", "start": "node src/subm.js", + "upsert": "node src/request1bld.js", "deploy": "gcloud app deploy", "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index cb1e002..bccf703 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -35,11 +35,18 @@ function getContent(host, path, headers, { get }) { }); } +const { keys } = Object; +const query = opts => + keys(opts).length > 0 ? `?${new URLSearchParams(opts).toString()}` : ''; + /** * Discord API (a small slice of it, anyway) * * @param {string} token - * @param {{ get: typeof import('https').get }} io + * @param {{ + * get: typeof import('https').get, + * setTimeout: typeof setTimeout, + * }} io * * // https://discordapp.com/developers/docs/resources/user * @typedef {{ @@ -76,21 +83,63 @@ function getContent(host, path, headers, { get }) { * }} UserObject * @typedef { string } Snowflake 64 bit numeral * @typedef { string } TimeStamp ISO8601 format + * + * https://discord.com/developers/docs/resources/channel#message-object + * @typedef {{ + * id: Snowflake, + * author: UserObject, + * content: string, + * timestamp: TimeStamp + * }} MessageObject */ -function DiscordAPI(token, { get }) { +function DiscordAPI(token, { get, setTimeout }) { // cribbed from rchain-dbr/o2r/gateway/server/main.js const host = 'discordapp.com'; const api = '/api/v6'; const headers = { Authorization: `Bot ${token}` }; + /** + * @param {string} path + * @returns {Promise} + */ const getJSON = async path => { + console.log('getJSON', { path }); const body = await getContent(host, path, headers, { get }); const data = JSON.parse(body); // console.log('Discord done:', Object.keys(data)); + if ('retry_after' in data) { + await new Promise(r => setTimeout(r, data.retry_after)); + return getJSON(path); + } return data; }; return freeze({ + /** @param { Snowflake } channelID */ + channels: channelID => { + return freeze({ + /** + * @param {Record} opts + * @returns {Promise} + */ + getMessages: (opts = {}) => + getJSON(`${api}/channels/${channelID}/messages${query(opts)}`), + /** @param { Snowflake } messageID */ + messages: messageID => + freeze({ + /** + * @param {string} emoji + * @returns {Promise} + */ + reactions: emoji => + getJSON( + `${api}/channels/${channelID}/messages/${messageID}/reactions/${encodeURIComponent( + emoji, + )}`, + ), + }), + }); + }, /** * @param { string } userID * @returns { Promise } @@ -120,8 +169,7 @@ function DiscordAPI(token, { get }) { const opts = after ? { limit: `${limit}`, after } : { limit: `${limit}` }; - const query = new URLSearchParams(opts).toString(); - return getJSON(`${api}/guilds/${guildID}/members?${query}`); + return getJSON(`${api}/guilds/${guildID}/members${query(opts)}`); }, }); }, @@ -136,6 +184,27 @@ function avatar(user) { return `${avatarBase}/${user.id}/${user.avatar}.png`; } +/** + * @param {(opts: any) => Promise} fn + * @param {number} [limit] + * @template {{ id: string }} T + */ +async function paged(fn, limit = 100) { + /** @type {T[][]} */ + const pages = []; + /** @type { string | undefined } */ + let before; + do { + console.error('getting page', pages.length, before); + // eslint-disable-next-line no-await-in-loop + const page = await fn(before ? { limit, before } : { limit }); + if (!page.length) break; + before = page.slice(-1)[0].id; + pages.push(page); + } while (before); + return pages.flat(); +} + /** * @param {ReturnType['guilds']>} guild */ @@ -163,11 +232,12 @@ async function pagedMembers(guild) { * @param {{ * get: typeof import('https').get, * stdout: typeof import('process').stdout + * setTimeout: typeof setTimeout, * }} io */ -async function main(env, { stdout, get }) { +async function integrationTest(env, { stdout, get, setTimeout }) { const config = makeConfig(env); - const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get }); + const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get, setTimeout }); const guild = discordAPI.guilds(config`DISCORD_GUILD_ID`); const roles = await guild.roles(); @@ -179,12 +249,15 @@ async function main(env, { stdout, get }) { /* global require, process */ if (require.main === module) { - main(process.env, { + integrationTest(process.env, { stdout: process.stdout, // eslint-disable-next-line global-require get: require('https').get, + // @ts-ignore + // eslint-disable-next-line no-undef + setTimeout, }).catch(err => console.error(err)); } /* global module */ -module.exports = { DiscordAPI, avatar }; +module.exports = { DiscordAPI, avatar, getContent, paged }; diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js new file mode 100644 index 0000000..a05d67a --- /dev/null +++ b/subm/src/request1bld.js @@ -0,0 +1,220 @@ +/* eslint-disable no-await-in-loop */ +// See https://github.com/Agoric/validator-profiles/wiki/Request-1-BLD + +const { DiscordAPI, getContent, paged } = require('./discordGuild.js'); +const { upsert } = require('./sheetAccess'); +const { searchBySender, transfers } = require('./tendermintRPC'); + +const config = { + host: 'rpc-agoric.nodes.guru', + address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h', + quorum: 2, +}; + +const fail = () => { + throw Error(); +}; + +/** + * @param {ReturnType['channels']>} channel + * @param {ReturnType['guilds']>} guild + * @param {Snowflake} role + * @param {number} quorum + * @yields {{ message: Message, endorsers: User[] }} + * @typedef {import('./discordGuild').Snowflake} Snowflake + */ +async function* authorizedRequests(channel, guild, role, quorum) { + /** @type {Map} */ + const memberDetail = new Map(); + /** @param {Snowflake} id */ + const getMemberDetail = async id => { + if (memberDetail.has(id)) { + return memberDetail.get(id) || fail(); + } + const detail = await guild.members(id); + // console.log(detail); + memberDetail.set(id, detail); + return detail; + }; + + const messages = await paged(channel.getMessages); + const hasAddr = messages.filter(msg => msg.content.match(/agoric1/)); + if (!hasAddr) return; + const hasChecks = hasAddr.filter(msg => { + const [checks] = (msg.reactions || []).filter(r => r.emoji.name === '✅'); + return (checks || {}).count >= quorum; + }); + if (!hasChecks) return; + + for (const msg of hasChecks) { + const endorsements = await channel.messages(msg.id).reactions('✅'); + const endorsers = []; + for (const endorsement of endorsements) { + const detail = await getMemberDetail(endorsement.id); + if (detail && detail.roles && detail.roles.includes(role)) { + endorsers.push(detail); + } + } + if (endorsers.length >= quorum) { + const [_, address] = msg.content.match(/(agoric1\S+)/); + if (typeof address !== 'string') throw TypeError(address); + yield { message: msg, address, endorsers }; + } + } +} + +/** + * @param {ReturnType['channels']>} channel + * @param {ReturnType['guilds']>} guild + * @param {string} roleID + * @param {{ + * get: typeof import('https').get, + * }} io + */ +async function requestStatus(channel, guild, roleID, { get }) { + const txs = await getContent( + config.host, + searchBySender(config.address), + {}, + { get }, + ).then(txt => JSON.parse(txt).result.txs); + + const txfrs = transfers(txs); + // console.log(txfrs); + const byRecipient = new Map(txfrs.map(txfr => [txfr.recipient, txfr])); + // console.log(byRecipient.keys()); + + const result = []; + const eachRequest = authorizedRequests(channel, guild, roleID, config.quorum); + for await (const { message, address, endorsers } of eachRequest) { + const hash = byRecipient.has(address) + ? byRecipient.get(address).hash + : undefined; + result.push({ + message, + address, + endorsers, + hash, + }); + } + return result; +} + +const label = user => `${user.username}#${user.discriminator}`; + +/** + * @param {Record} env + * @param {{ + * get: typeof import('https').get, + * setTimeout: typeof setTimeout, + * }} io + */ +async function main(env, { get, setTimeout }) { + const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout }); + const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); + + // to get mod-1-bld role id: + // console.log(await guild.roles()); + + const channel = discordAPI.channels(env.CHANNEL_ID); + + const txs = await getContent( + config.host, + searchBySender(config.address), + {}, + { get }, + ).then(txt => JSON.parse(txt).result.txs); + + const txfrs = transfers(txs); + // console.log(txfrs); + const byRecipient = new Map(txfrs.map(txfr => [txfr.recipient, txfr])); + // console.log(byRecipient.keys()); + + const header = [ + 'timestamp', + 'msgID', + 'requestor', + 'address', + 'endorsers', + 'hash', + ]; + console.log(header.join(',')); + for await (const { message: msg, address, endorsers } of authorizedRequests( + channel, + guild, + env.REVIEWER_ROLE_ID, + 2, + )) { + const ok = endorsers.map(u => label(u.user)).join(' '); + const hash = byRecipient.has(address) ? byRecipient.get(address).hash : ''; + console.log( + `${msg.timestamp},${msg.id},${label( + msg.author, + )},${address},${ok},${hash}`, + ); + } +} + +/** + * @param {string[]} args + * @param {Record} env + * @param {Object} io + * @param {typeof import('google-spreadsheet').GoogleSpreadsheet} io.GoogleSpreadsheet + * @param {typeof import('https').get} io.get + * @param {typeof setTimeout} io.setTimeout + */ +const main2 = async (args, env, { get, setTimeout, GoogleSpreadsheet }) => { + const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout }); + const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); + + // to get mod-1-bld role id: + // console.log(await guild.roles()); + + const channel = discordAPI.channels(env.CHANNEL_ID); + + const creds = { + client_email: env.GOOGLE_SERVICES_EMAIL, + private_key: env.GCS_PRIVATE_KEY, + }; + // Initialize the sheet - doc ID is the long id in the sheets URL + const doc = new GoogleSpreadsheet(env.SHEET_1BLD_ID); + // Initialize Auth - see https://theoephraim.github.io/node-google-spreadsheet/#/getting-started/authentication + await doc.useServiceAccountAuth(creds); + await doc.loadInfo(); // loads document properties and worksheets + console.log(doc.title); + const sheet = doc.sheetsByIndex[0]; + + const memberLabel = mem => mem.nick || label(mem.user); + const eachStatus = await requestStatus(channel, guild, env.REVIEWER_ROLE_ID, { + get, + }); + for (const { message: msg, address, endorsers, hash } of eachStatus) { + const record = { + To: address, + Link: `https://discord.com/channels/${env.DISCORD_GUILD_ID}/${env.CHANNEL_ID}/${msg.id}`, + At: msg.timestamp.slice(0, '1999-01-01T12:59'.length).replace('T', ' '), + By: label(msg.author), + Reviewers: endorsers.map(memberLabel).join(','), + Tx: hash && `https://agoric.bigdipper.live/transactions/${hash}`, + Request: msg.content, + }; + await upsert(sheet, address, record); // do not interleave upserts! + } +}; + +/* global require, process, module */ +if (require.main === module) { + main2( + process.argv.slice(2), + { ...process.env }, + { + // eslint-disable-next-line global-require + get: require('https').get, + setTimeout, + // eslint-disable-next-line global-require + GoogleSpreadsheet: require('google-spreadsheet').GoogleSpreadsheet, // please excuse CJS + }, + ).catch(err => console.error(err)); +} + +module.exports = { requestStatus }; diff --git a/subm/src/subm.js b/subm/src/subm.js index 0928725..00bd59a 100644 --- a/subm/src/subm.js +++ b/subm/src/subm.js @@ -19,6 +19,7 @@ const { makeFirebaseAdmin, getFirebaseConfig } = require('./firebaseTool.js'); const { generateV4SignedPolicy } = require('./objStore.js'); const { lookup, upsert } = require('./sheetAccess.js'); const { makeConfig } = require('./config.js'); +const { requestStatus } = require('./request1bld.js'); const { freeze, keys, values, entries } = Object; // please excuse freeze vs. harden @@ -72,6 +73,7 @@ const Site = freeze({ callback: '/auth/discord/callback', badLogin: '/loginRefused', contactForm: '/community/contact', + request1bld: '/community/request1bld', uploadSlog: '/participant/slogForm', uploadSuccess: '/participant/slogOK', loadGenKey: '/participant/loadGenKey', @@ -218,6 +220,40 @@ ${Site.welcome(member)} `, + + exploreAddr: addr => + `${addr}`, + exploreTx: hash => + hash + ? `${hash.slice(0, 16)}...` + : '', + + /** @param {Awaited>} requests */ + requestStatus: requests => `${AgoricStyle.top} +

1 BLD Request Status

+
background: Request 1 BLD
+ + + + ${requests + .map( + ({ message: { id, author, timestamp }, address, hash }) => ` + + + + + + `, + ) + .join('\n')} + +
RequestToTx?
+ + ${timestamp.slice(0, '1999-01-01T12:59'.length).replace('T', ' ')} + ${(author || {}).username || 'author'} + ${Site.exploreAddr(address)}${Site.exploreTx(hash)}
+ `, /** * @param { GuildMember } member * @param { string } combinedToken @@ -517,6 +553,7 @@ function makeDiscordBot(guild, authorizedRoles, opts, powers) { * @param {{ * clock: () => number, * get: typeof import('https').get, + * setTimeout: typeof setTimeout, * express: typeof import('express'), * GoogleSpreadsheet: typeof import('google-spreadsheet').GoogleSpreadsheet, * makeStorage: (...args: unknown[]) => StorageT, @@ -526,7 +563,7 @@ function makeDiscordBot(guild, authorizedRoles, opts, powers) { */ async function main( env, - { clock, get, express, makeStorage, admin, GoogleSpreadsheet }, + { clock, get, setTimeout, express, makeStorage, admin, GoogleSpreadsheet }, ) { const app = express(); app.enable('trust proxy'); // trust X-Forwarded-* headers @@ -555,7 +592,7 @@ async function main( }); await doc.loadInfo(); - const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get }); + const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get, setTimeout }); const guild = discordAPI.guilds(config`DISCORD_GUILD_ID`); const loadGenAdmin = makeFirebaseAdmin(admin, getFirebaseConfig(config)); loadGenAdmin.init(); @@ -643,6 +680,17 @@ async function main( ); }); + const channel = discordAPI.channels(config`CHANNEL_ID`); + const roleID = config`REVIEWER_ROLE_ID`; + app.get(Site.path.request1bld, async (req, res) => { + try { + const status = await requestStatus(channel, guild, roleID, { get }); + res.send(Site.requestStatus(status)); + } catch (err) { + handleError(res, req.baseUrl, err); + } + }); + // Upload form // Note the actual upload request goes directly to Google Cloud Storage. app.get( @@ -703,6 +751,7 @@ if (require.main === module) { clock: () => Date.now(), express: require('express'), get: require('https').get, + setTimeout, makeStorage: (C => (...args) => new C(...args))( require('@google-cloud/storage').Storage, ), diff --git a/subm/src/tendermintRPC.js b/subm/src/tendermintRPC.js new file mode 100644 index 0000000..c487e47 --- /dev/null +++ b/subm/src/tendermintRPC.js @@ -0,0 +1,55 @@ +const { getContent } = require('./discordGuild.js'); + +const { fromEntries } = Object; + +const config = { + host: 'rpc-agoric.nodes.guru', + address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h', +}; + +const searchBySender = address => + `/tx_search?query="transfer.sender='${address}'"&per_page=100`; + +/** + * @param {{ hash: string, tx_result: { log: string }}[]} txs + * @returns {{ hash: string, recipient: string, sender: string, amount: string}[]} + */ +const transfers = txs => + txs + .map(({ hash, tx_result: { log: logText } }) => { + const [{ events }] = JSON.parse(logText); + if (!events) return []; + return events + .filter(({ type }) => type === 'transfer') + .map(({ attributes }) => ({ + hash, + ...fromEntries(attributes.map(({ key, value }) => [key, value])), + })); + }) + .flat(); + +/** + * @param {{ + * get: typeof import('https').get, + * }} io + */ +const main = async ({ get }) => { + const txs = await getContent( + config.host, + searchBySender(config.address), + {}, + { get }, + ).then(txt => JSON.parse(txt).result.txs); + + console.log(transfers(txs.slice(0, 3))); +}; + +/* global require, module */ +if (require.main === module) { + main({ + // eslint-disable-next-line global-require + get: require('https').get, + }).catch(err => console.error(err)); +} + +module.exports = { searchBySender, transfers };