From a61de7c52419d001792e3be42df08c5a02696ff2 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 19 Mar 2022 10:38:12 -0500 Subject: [PATCH 01/14] test(request1bld): find endorsements on recent relevant messages This is an integration test, exploring the Discord REST API. --- subm/src/discordGuild.js | 12 +++++++++ subm/src/request1bld.js | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 subm/src/request1bld.js diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index cb1e002..590cfac 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -91,6 +91,18 @@ function DiscordAPI(token, { get }) { }; return freeze({ + channels: channelID => { + return freeze({ + getMessages: () => getJSON(`${api}/channels/${channelID}/messages`), + messages: messageID => + freeze({ + reactions: emoji => + getJSON( + `${api}/channels/${channelID}/messages/${messageID}/reactions/${emoji}`, + ), + }), + }); + }, /** * @param { string } userID * @returns { Promise } diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js new file mode 100644 index 0000000..950e637 --- /dev/null +++ b/subm/src/request1bld.js @@ -0,0 +1,58 @@ +/* eslint-disable no-await-in-loop */ +// See https://github.com/Agoric/validator-profiles/wiki/Request-1-BLD + +const { DiscordAPI } = require('./discordGuild'); + +const ADMIN_ROLE_ID = '412648251196702741'; + +/** + * @param {Record} env + * @param {{ + * get: typeof import('https').get, + * stdout: typeof import('process').stdout + * }} io + */ +async function main(env, { stdout, get }) { + const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get }); + const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); + + const channel = discordAPI.channels(env.CHANNEL_ID); + const messages = await channel.getMessages(); + for (const message of messages) { + if (message.content.match(/agoric1/)) { + console.log(message); + console.log(message.reactions); + for (const reaction of message.reactions) { + if (reaction.emoji.name === '✅') { + console.log('endorsement!!'); + const endorsements = await channel + .messages(message.id) + .reactions(encodeURIComponent('✅')); + console.log(endorsements); + for (const endorsement of endorsements) { + console.log( + `${message.content} endorsed by ${endorsement.username}`, + ); + const detail = await guild.members(endorsement.id); + console.log({ detail }); + if (detail.roles.includes(ADMIN_ROLE_ID)) { + console.log('endorsed by admin!'); + } + } + } + } + } + } + + // const roles = await guild.roles(); + // console.log(roles); +} + +/* global require, process */ +if (require.main === module) { + main(process.env, { + stdout: process.stdout, + // eslint-disable-next-line global-require + get: require('https').get, + }).catch(err => console.error(err)); +} From fca01c62bc1b4f99075764aec824836002cd59aa Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 19 Mar 2022 11:24:53 -0500 Subject: [PATCH 02/14] refactor(request1bld): factor authorizedRequests() out of main() --- subm/src/discordGuild.js | 4 ++- subm/src/request1bld.js | 65 ++++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index 590cfac..f86fabd 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -98,7 +98,9 @@ function DiscordAPI(token, { get }) { freeze({ reactions: emoji => getJSON( - `${api}/channels/${channelID}/messages/${messageID}/reactions/${emoji}`, + `${api}/channels/${channelID}/messages/${messageID}/reactions/${encodeURIComponent( + emoji, + )}`, ), }), }); diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 950e637..8ff63d5 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -5,6 +5,38 @@ const { DiscordAPI } = require('./discordGuild'); const ADMIN_ROLE_ID = '412648251196702741'; +async function* authorizedRequests(channel, guild, role, quorum) { + const memberDetail = new Map(); + const getMemberDetail = async id => { + if (memberDetail.has(id)) { + return memberDetail.get(id); + } + const detail = await guild.members(id); + memberDetail.set(id, detail); + return detail; + }; + + const messages = await channel.getMessages(); + const hasAddr = messages.filter(msg => msg.content.match(/agoric1/)); + const hasChecks = hasAddr.filter( + msg => msg.reactions.filter(r => r.emoji.name === '✅').length >= quorum, + ); + + for (const msg of hasChecks) { + const endorsements = await channel.messages(msg.id).reactions('✅'); + const endorsers = []; + for (const endorsement of endorsements) { + const detail = await getMemberDetail(endorsement.id); + if (detail.roles.includes(role)) { + endorsers.push(detail); + } + } + if (endorsers.length >= quorum) { + yield { message: msg, endorsers }; + } + } +} + /** * @param {Record} env * @param {{ @@ -17,38 +49,13 @@ async function main(env, { stdout, get }) { const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); const channel = discordAPI.channels(env.CHANNEL_ID); - const messages = await channel.getMessages(); - for (const message of messages) { - if (message.content.match(/agoric1/)) { - console.log(message); - console.log(message.reactions); - for (const reaction of message.reactions) { - if (reaction.emoji.name === '✅') { - console.log('endorsement!!'); - const endorsements = await channel - .messages(message.id) - .reactions(encodeURIComponent('✅')); - console.log(endorsements); - for (const endorsement of endorsements) { - console.log( - `${message.content} endorsed by ${endorsement.username}`, - ); - const detail = await guild.members(endorsement.id); - console.log({ detail }); - if (detail.roles.includes(ADMIN_ROLE_ID)) { - console.log('endorsed by admin!'); - } - } - } - } - } - } - // const roles = await guild.roles(); - // console.log(roles); + for await (const x of authorizedRequests(channel, guild, ADMIN_ROLE_ID, 1)) { + console.log(x); + } } -/* global require, process */ +/* global require, process, module */ if (require.main === module) { main(process.env, { stdout: process.stdout, From 9110ff7dd5a5b1dc8cbcb5797e6bfc1add2e36f9 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 19 Mar 2022 11:36:27 -0500 Subject: [PATCH 03/14] feat(request1bld): report recent authorized requests in CSV format --- subm/src/request1bld.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 8ff63d5..bbe5d87 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -50,8 +50,20 @@ async function main(env, { stdout, get }) { const channel = discordAPI.channels(env.CHANNEL_ID); - for await (const x of authorizedRequests(channel, guild, ADMIN_ROLE_ID, 1)) { - console.log(x); + const header = ['timestamp', 'msgID', 'requestor', 'address', 'endorsers']; + console.log(header.join(',')); + for await (const { message: msg, endorsers } of authorizedRequests( + channel, + guild, + ADMIN_ROLE_ID, + 1, + )) { + const [_, address] = msg.content.match(/(agoric1\S+)/); + const label = user => `${user.username}#${user.discriminator}`; + const ok = endorsers.map(u => label(u.user)).join(' '); + console.log( + `${msg.timestamp},${msg.id},${label(msg.author)},${address},${ok}`, + ); } } From 411f5acf1c4951358edf5d3ad6e9e4b0df91e815 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 19 Mar 2022 11:53:54 -0500 Subject: [PATCH 04/14] fix(request-1-bld): handle missing reactions, quorum = 2 --- subm/src/request1bld.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index bbe5d87..79af860 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -3,8 +3,6 @@ const { DiscordAPI } = require('./discordGuild'); -const ADMIN_ROLE_ID = '412648251196702741'; - async function* authorizedRequests(channel, guild, role, quorum) { const memberDetail = new Map(); const getMemberDetail = async id => { @@ -12,15 +10,19 @@ async function* authorizedRequests(channel, guild, role, quorum) { return memberDetail.get(id); } const detail = await guild.members(id); + // console.log(detail); memberDetail.set(id, detail); return detail; }; const messages = await channel.getMessages(); const hasAddr = messages.filter(msg => msg.content.match(/agoric1/)); - const hasChecks = hasAddr.filter( - msg => msg.reactions.filter(r => r.emoji.name === '✅').length >= quorum, - ); + if (!hasAddr) return; + const hasChecks = hasAddr.filter(msg => { + const [checks] = (msg.reactions || []).filter(r => r.emoji.name === '✅'); + return (checks || {}).count >= quorum; + }); + if (!hasChecks) return; for (const msg of hasChecks) { const endorsements = await channel.messages(msg.id).reactions('✅'); @@ -48,6 +50,9 @@ async function main(env, { stdout, get }) { const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get }); const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); + // to get mod-1-bld role id: + // console.log(await guild.roles()); + const channel = discordAPI.channels(env.CHANNEL_ID); const header = ['timestamp', 'msgID', 'requestor', 'address', 'endorsers']; @@ -55,8 +60,8 @@ async function main(env, { stdout, get }) { for await (const { message: msg, endorsers } of authorizedRequests( channel, guild, - ADMIN_ROLE_ID, - 1, + env.REVIEWER_ROLE_ID, + 2, )) { const [_, address] = msg.content.match(/(agoric1\S+)/); const label = user => `${user.username}#${user.discriminator}`; From 7db34e2249c1786f9028e42fec83e7d4713d5c37 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 22 Mar 2022 20:31:11 -0500 Subject: [PATCH 05/14] feat(request1bld): scan 100 messages rather than just 50 --- subm/src/discordGuild.js | 8 +++++--- subm/src/request1bld.js | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index f86fabd..af1766e 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -35,6 +35,8 @@ function getContent(host, path, headers, { get }) { }); } +const query = opts => (opts ? `?${new URLSearchParams(opts).toString()}` : ''); + /** * Discord API (a small slice of it, anyway) * @@ -93,7 +95,8 @@ function DiscordAPI(token, { get }) { return freeze({ channels: channelID => { return freeze({ - getMessages: () => getJSON(`${api}/channels/${channelID}/messages`), + getMessages: opts => + getJSON(`${api}/channels/${channelID}/messages${query(opts)}`), messages: messageID => freeze({ reactions: emoji => @@ -134,8 +137,7 @@ function DiscordAPI(token, { get }) { const opts = after ? { limit: `${limit}`, after } : { limit: `${limit}` }; - const query = new URLSearchParams(opts).toString(); - return getJSON(`${api}/guilds/${guildID}/members?${query}`); + return getJSON(`${api}/guilds/${guildID}/members${query(opts)}`); }, }); }, diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 79af860..07ac280 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -3,11 +3,25 @@ const { DiscordAPI } = require('./discordGuild'); +const fail = () => { + throw Error(); +}; + +/** + * @param {ReturnType['channels']>} channel + * @param {ReturnType['guilds']>} guild + * @param {Snowflake} role + * @param {number} quorum + * @yields {{ message: Message, endorsers: User[] }} + * @typedef {import('./discordGuild').Snowflake} Snowflake + */ async function* authorizedRequests(channel, guild, role, quorum) { + /** @type {Map} */ const memberDetail = new Map(); + /** @param {Snowflake} id */ const getMemberDetail = async id => { if (memberDetail.has(id)) { - return memberDetail.get(id); + return memberDetail.get(id) || fail(); } const detail = await guild.members(id); // console.log(detail); @@ -15,7 +29,7 @@ async function* authorizedRequests(channel, guild, role, quorum) { return detail; }; - const messages = await channel.getMessages(); + const messages = await channel.getMessages({ limit: 100 }); const hasAddr = messages.filter(msg => msg.content.match(/agoric1/)); if (!hasAddr) return; const hasChecks = hasAddr.filter(msg => { From 4dde767304a5204053892dfd032f706e77c13b98 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 13:50:17 -0500 Subject: [PATCH 06/14] feat(request1bld): handle rate limiting with retry_after --- subm/src/discordGuild.js | 19 +++++++++++++++---- subm/src/request1bld.js | 6 ++++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index af1766e..d85c6c7 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -41,7 +41,10 @@ const query = opts => (opts ? `?${new URLSearchParams(opts).toString()}` : ''); * Discord API (a small slice of it, anyway) * * @param {string} token - * @param {{ get: typeof import('https').get }} io + * @param {{ + * get: typeof import('https').get, + * setTimeout: typeof setTimeout, + * }} io * * // https://discordapp.com/developers/docs/resources/user * @typedef {{ @@ -79,7 +82,7 @@ const query = opts => (opts ? `?${new URLSearchParams(opts).toString()}` : ''); * @typedef { string } Snowflake 64 bit numeral * @typedef { string } TimeStamp ISO8601 format */ -function DiscordAPI(token, { get }) { +function DiscordAPI(token, { get, setTimeout }) { // cribbed from rchain-dbr/o2r/gateway/server/main.js const host = 'discordapp.com'; const api = '/api/v6'; @@ -89,6 +92,10 @@ function DiscordAPI(token, { get }) { const body = await getContent(host, path, headers, { get }); const data = JSON.parse(body); // console.log('Discord done:', Object.keys(data)); + if ('retry_after' in data) { + await new Promise(r => setTimeout(r, data.retry_after)); + return getJSON(path); + } return data; }; @@ -179,11 +186,12 @@ async function pagedMembers(guild) { * @param {{ * get: typeof import('https').get, * stdout: typeof import('process').stdout + * setTimeout: typeof setTimeout, * }} io */ -async function main(env, { stdout, get }) { +async function main(env, { stdout, get, setTimeout }) { const config = makeConfig(env); - const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get }); + const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get, setTimeout }); const guild = discordAPI.guilds(config`DISCORD_GUILD_ID`); const roles = await guild.roles(); @@ -199,6 +207,9 @@ if (require.main === module) { stdout: process.stdout, // eslint-disable-next-line global-require get: require('https').get, + // @ts-ignore + // eslint-disable-next-line no-undef + setTimeout, }).catch(err => console.error(err)); } diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 07ac280..005c939 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -58,10 +58,11 @@ async function* authorizedRequests(channel, guild, role, quorum) { * @param {{ * get: typeof import('https').get, * stdout: typeof import('process').stdout + * setTimeout: typeof setTimeout, * }} io */ -async function main(env, { stdout, get }) { - const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get }); +async function main(env, { stdout, get, setTimeout }) { + const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout }); const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); // to get mod-1-bld role id: @@ -92,5 +93,6 @@ if (require.main === module) { stdout: process.stdout, // eslint-disable-next-line global-require get: require('https').get, + setTimeout, }).catch(err => console.error(err)); } From 243a315e0e42c80a92d2d6d5541d99736e75312e Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 18:35:32 -0500 Subject: [PATCH 07/14] test(request1bld): fetch transactions from an address --- subm/src/discordGuild.js | 2 +- subm/src/tendermintRPC.js | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 subm/src/tendermintRPC.js diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index d85c6c7..c19e5d1 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -214,4 +214,4 @@ if (require.main === module) { } /* global module */ -module.exports = { DiscordAPI, avatar }; +module.exports = { DiscordAPI, avatar, getContent }; diff --git a/subm/src/tendermintRPC.js b/subm/src/tendermintRPC.js new file mode 100644 index 0000000..6681434 --- /dev/null +++ b/subm/src/tendermintRPC.js @@ -0,0 +1,33 @@ +const { getContent } = require('./discordGuild.js'); + +const searchBySender = address => + `/tx_search?query="transfer.sender='${address}'"`; + +const config = { + host: 'rpc-agoric.nodes.guru', + address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h', +}; + +/** + * @param {{ + * get: typeof import('https').get, + * }} io + */ +const main = async ({ get }) => { + const txt = await getContent( + config.host, + searchBySender(config.address), + {}, + { get }, + ); + const data = JSON.parse(txt); + console.log(data); +}; + +/* global require, module */ +if (require.main === module) { + main({ + // eslint-disable-next-line global-require + get: require('https').get, + }).catch(err => console.error(err)); +} From 3456a8480770ffc2b75f041ad420f123f13b1fbb Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 19:13:31 -0500 Subject: [PATCH 08/14] test: extract transfers recipients from tx_search results --- subm/src/tendermintRPC.js | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/subm/src/tendermintRPC.js b/subm/src/tendermintRPC.js index 6681434..706ac49 100644 --- a/subm/src/tendermintRPC.js +++ b/subm/src/tendermintRPC.js @@ -1,27 +1,43 @@ const { getContent } = require('./discordGuild.js'); -const searchBySender = address => - `/tx_search?query="transfer.sender='${address}'"`; +const { fromEntries } = Object; const config = { host: 'rpc-agoric.nodes.guru', address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h', }; +const searchBySender = address => + `/tx_search?query="transfer.sender='${address}'"`; + +const transfers = txs => + txs + .map(({ hash, tx_result: { log: logText } }) => { + const [{ events }] = JSON.parse(logText); + if (!events) return []; + return events + .filter(({ type }) => type === 'transfer') + .map(({ attributes }) => ({ + hash, + ...fromEntries(attributes.map(({ key, value }) => [key, value])), + })); + }) + .flat(); + /** * @param {{ * get: typeof import('https').get, * }} io */ const main = async ({ get }) => { - const txt = await getContent( + const txs = await getContent( config.host, searchBySender(config.address), {}, { get }, - ); - const data = JSON.parse(txt); - console.log(data); + ).then(txt => JSON.parse(txt).result.txs); + + console.log(transfers(txs.slice(0, 3))); }; /* global require, module */ From 2ccee05f38bf4de9c10916ff4a86b547a93afc22 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 19:20:47 -0500 Subject: [PATCH 09/14] chore(request1bld): never mind stdout --- subm/src/request1bld.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 005c939..bf5dfaa 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -57,11 +57,10 @@ async function* authorizedRequests(channel, guild, role, quorum) { * @param {Record} env * @param {{ * get: typeof import('https').get, - * stdout: typeof import('process').stdout * setTimeout: typeof setTimeout, * }} io */ -async function main(env, { stdout, get, setTimeout }) { +async function main(env, { get, setTimeout }) { const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout }); const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); @@ -90,7 +89,6 @@ async function main(env, { stdout, get, setTimeout }) { /* global require, process, module */ if (require.main === module) { main(process.env, { - stdout: process.stdout, // eslint-disable-next-line global-require get: require('https').get, setTimeout, From e41ea271ee6e87bfd6640254b04dd5434dd6da11 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 19:54:54 -0500 Subject: [PATCH 10/14] feat(request1bld): mix tx hash in with approved requests --- subm/src/request1bld.js | 40 +++++++++++++++++++++++++++++++++------ subm/src/tendermintRPC.js | 4 +++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index bf5dfaa..9383d95 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -1,7 +1,13 @@ /* eslint-disable no-await-in-loop */ // See https://github.com/Agoric/validator-profiles/wiki/Request-1-BLD -const { DiscordAPI } = require('./discordGuild'); +const { DiscordAPI, getContent } = require('./discordGuild'); +const { searchBySender, transfers } = require('./tendermintRPC'); + +const config = { + host: 'rpc-agoric.nodes.guru', + address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h', +}; const fail = () => { throw Error(); @@ -48,7 +54,8 @@ async function* authorizedRequests(channel, guild, role, quorum) { } } if (endorsers.length >= quorum) { - yield { message: msg, endorsers }; + const [_, address] = msg.content.match(/(agoric1\S+)/); + yield { message: msg, address, endorsers }; } } } @@ -69,19 +76,40 @@ async function main(env, { get, setTimeout }) { const channel = discordAPI.channels(env.CHANNEL_ID); - const header = ['timestamp', 'msgID', 'requestor', 'address', 'endorsers']; + const txs = await getContent( + config.host, + searchBySender(config.address), + {}, + { get }, + ).then(txt => JSON.parse(txt).result.txs); + + const txfrs = transfers(txs); + // console.log(txfrs); + const byRecipient = new Map(txfrs.map(txfr => [txfr.recipient, txfr])); + // console.log(byRecipient.keys()); + + const header = [ + 'timestamp', + 'msgID', + 'requestor', + 'address', + 'endorsers', + 'hash', + ]; console.log(header.join(',')); - for await (const { message: msg, endorsers } of authorizedRequests( + for await (const { message: msg, address, endorsers } of authorizedRequests( channel, guild, env.REVIEWER_ROLE_ID, 2, )) { - const [_, address] = msg.content.match(/(agoric1\S+)/); const label = user => `${user.username}#${user.discriminator}`; const ok = endorsers.map(u => label(u.user)).join(' '); + const hash = byRecipient.has(address) ? byRecipient.get(address).hash : ''; console.log( - `${msg.timestamp},${msg.id},${label(msg.author)},${address},${ok}`, + `${msg.timestamp},${msg.id},${label( + msg.author, + )},${address},${ok},${hash}`, ); } } diff --git a/subm/src/tendermintRPC.js b/subm/src/tendermintRPC.js index 706ac49..fdb1bcf 100644 --- a/subm/src/tendermintRPC.js +++ b/subm/src/tendermintRPC.js @@ -8,7 +8,7 @@ const config = { }; const searchBySender = address => - `/tx_search?query="transfer.sender='${address}'"`; + `/tx_search?query="transfer.sender='${address}'"&per_page=100`; const transfers = txs => txs @@ -47,3 +47,5 @@ if (require.main === module) { get: require('https').get, }).catch(err => console.error(err)); } + +module.exports = { searchBySender, transfers }; From ca26fb250793e291888a908e0d11ef04afd7c66b Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 21:16:02 -0500 Subject: [PATCH 11/14] feat(subm): request1bld status page --- subm/src/discordGuild.js | 20 +++++++++++++++ subm/src/request1bld.js | 44 ++++++++++++++++++++++++++++++++ subm/src/subm.js | 53 +++++++++++++++++++++++++++++++++++++-- subm/src/tendermintRPC.js | 4 +++ 4 files changed, 119 insertions(+), 2 deletions(-) diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index c19e5d1..5bc50a5 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -81,6 +81,14 @@ const query = opts => (opts ? `?${new URLSearchParams(opts).toString()}` : ''); * }} UserObject * @typedef { string } Snowflake 64 bit numeral * @typedef { string } TimeStamp ISO8601 format + * + * https://discord.com/developers/docs/resources/channel#message-object + * @typedef {{ + * id: Snowflake, + * author: UserObject, + * content: string, + * timestamp: TimeStamp + * }} MessageObject */ function DiscordAPI(token, { get, setTimeout }) { // cribbed from rchain-dbr/o2r/gateway/server/main.js @@ -88,6 +96,10 @@ function DiscordAPI(token, { get, setTimeout }) { const api = '/api/v6'; const headers = { Authorization: `Bot ${token}` }; + /** + * @param {string} path + * @returns {Promise} + */ const getJSON = async path => { const body = await getContent(host, path, headers, { get }); const data = JSON.parse(body); @@ -102,10 +114,18 @@ function DiscordAPI(token, { get, setTimeout }) { return freeze({ channels: channelID => { return freeze({ + /** + * @param {Record} opts + * @returns {Promise} + */ getMessages: opts => getJSON(`${api}/channels/${channelID}/messages${query(opts)}`), messages: messageID => freeze({ + /** + * @param {string} emoji + * @returns {Promise} + */ reactions: emoji => getJSON( `${api}/channels/${channelID}/messages/${messageID}/reactions/${encodeURIComponent( diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 9383d95..4aac3bd 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -7,6 +7,7 @@ const { searchBySender, transfers } = require('./tendermintRPC'); const config = { host: 'rpc-agoric.nodes.guru', address: 'agoric15qxmfufeyj4zm9zwnsczp72elxsjsvd0vm4q8h', + quorum: 2, }; const fail = () => { @@ -55,11 +56,52 @@ async function* authorizedRequests(channel, guild, role, quorum) { } if (endorsers.length >= quorum) { const [_, address] = msg.content.match(/(agoric1\S+)/); + if (typeof address !== 'string') throw TypeError(address); yield { message: msg, address, endorsers }; } } } +/** + * @param {ReturnType['channels']>} channel + * @param {ReturnType['guilds']>} guild + * @param {string} roleID + * @param {{ + * get: typeof import('https').get, + * }} io + */ +async function requestStatus(channel, guild, roleID, { get }) { + const txs = await getContent( + config.host, + searchBySender(config.address), + {}, + { get }, + ).then(txt => JSON.parse(txt).result.txs); + + const txfrs = transfers(txs); + // console.log(txfrs); + const byRecipient = new Map(txfrs.map(txfr => [txfr.recipient, txfr])); + // console.log(byRecipient.keys()); + + const result = []; + for await (const { + message: { id, timestamp, author }, + address, + endorsers, + } of authorizedRequests(channel, guild, roleID, config.quorum)) { + const hash = byRecipient.has(address) + ? byRecipient.get(address).hash + : undefined; + result.push({ + message: { id, timestamp, author }, + address, + endorsers, + hash, + }); + } + return result; +} + /** * @param {Record} env * @param {{ @@ -122,3 +164,5 @@ if (require.main === module) { setTimeout, }).catch(err => console.error(err)); } + +module.exports = { requestStatus }; diff --git a/subm/src/subm.js b/subm/src/subm.js index 0928725..00bd59a 100644 --- a/subm/src/subm.js +++ b/subm/src/subm.js @@ -19,6 +19,7 @@ const { makeFirebaseAdmin, getFirebaseConfig } = require('./firebaseTool.js'); const { generateV4SignedPolicy } = require('./objStore.js'); const { lookup, upsert } = require('./sheetAccess.js'); const { makeConfig } = require('./config.js'); +const { requestStatus } = require('./request1bld.js'); const { freeze, keys, values, entries } = Object; // please excuse freeze vs. harden @@ -72,6 +73,7 @@ const Site = freeze({ callback: '/auth/discord/callback', badLogin: '/loginRefused', contactForm: '/community/contact', + request1bld: '/community/request1bld', uploadSlog: '/participant/slogForm', uploadSuccess: '/participant/slogOK', loadGenKey: '/participant/loadGenKey', @@ -218,6 +220,40 @@ ${Site.welcome(member)} `, + + exploreAddr: addr => + `${addr}`, + exploreTx: hash => + hash + ? `${hash.slice(0, 16)}...` + : '', + + /** @param {Awaited>} requests */ + requestStatus: requests => `${AgoricStyle.top} +

1 BLD Request Status

+
background: Request 1 BLD
+ + + + ${requests + .map( + ({ message: { id, author, timestamp }, address, hash }) => ` + + + + + + `, + ) + .join('\n')} + +
RequestToTx?
+ + ${timestamp.slice(0, '1999-01-01T12:59'.length).replace('T', ' ')} + ${(author || {}).username || 'author'} + ${Site.exploreAddr(address)}${Site.exploreTx(hash)}
+ `, /** * @param { GuildMember } member * @param { string } combinedToken @@ -517,6 +553,7 @@ function makeDiscordBot(guild, authorizedRoles, opts, powers) { * @param {{ * clock: () => number, * get: typeof import('https').get, + * setTimeout: typeof setTimeout, * express: typeof import('express'), * GoogleSpreadsheet: typeof import('google-spreadsheet').GoogleSpreadsheet, * makeStorage: (...args: unknown[]) => StorageT, @@ -526,7 +563,7 @@ function makeDiscordBot(guild, authorizedRoles, opts, powers) { */ async function main( env, - { clock, get, express, makeStorage, admin, GoogleSpreadsheet }, + { clock, get, setTimeout, express, makeStorage, admin, GoogleSpreadsheet }, ) { const app = express(); app.enable('trust proxy'); // trust X-Forwarded-* headers @@ -555,7 +592,7 @@ async function main( }); await doc.loadInfo(); - const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get }); + const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get, setTimeout }); const guild = discordAPI.guilds(config`DISCORD_GUILD_ID`); const loadGenAdmin = makeFirebaseAdmin(admin, getFirebaseConfig(config)); loadGenAdmin.init(); @@ -643,6 +680,17 @@ async function main( ); }); + const channel = discordAPI.channels(config`CHANNEL_ID`); + const roleID = config`REVIEWER_ROLE_ID`; + app.get(Site.path.request1bld, async (req, res) => { + try { + const status = await requestStatus(channel, guild, roleID, { get }); + res.send(Site.requestStatus(status)); + } catch (err) { + handleError(res, req.baseUrl, err); + } + }); + // Upload form // Note the actual upload request goes directly to Google Cloud Storage. app.get( @@ -703,6 +751,7 @@ if (require.main === module) { clock: () => Date.now(), express: require('express'), get: require('https').get, + setTimeout, makeStorage: (C => (...args) => new C(...args))( require('@google-cloud/storage').Storage, ), diff --git a/subm/src/tendermintRPC.js b/subm/src/tendermintRPC.js index fdb1bcf..c487e47 100644 --- a/subm/src/tendermintRPC.js +++ b/subm/src/tendermintRPC.js @@ -10,6 +10,10 @@ const config = { const searchBySender = address => `/tx_search?query="transfer.sender='${address}'"&per_page=100`; +/** + * @param {{ hash: string, tx_result: { log: string }}[]} txs + * @returns {{ hash: string, recipient: string, sender: string, amount: string}[]} + */ const transfers = txs => txs .map(({ hash, tx_result: { log: logText } }) => { From 6c18f046391e27f7fe2366cf924db20423fbbd67 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 23 Mar 2022 21:36:59 -0500 Subject: [PATCH 12/14] docs(request1bld): document CHANNEL_ID, REVIEWER_ROLE_ID --- subm/app.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/subm/app.yaml b/subm/app.yaml index ba9a882..5baa7d8 100644 --- a/subm/app.yaml +++ b/subm/app.yaml @@ -31,7 +31,10 @@ includes: # # Discord # DISCORD_API_TOKEN: sekretsekret # DISCORD_GUILD_ID: 585... -# # DISCORD_USER_ID: 358096357862408195 +# # validator-1-bld +# CHANNEL_ID: 946... +# # mod1bld +# REVIEWER_ROLE_ID: 946... # DISCORD_CLIENT_ID: 874... # DISCORD_CLIENT_SECRET: sekretsekret From 53a31a73d33d8b9d5d222e787b52f3abd26d3b6c Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 2 Apr 2022 00:34:58 -0500 Subject: [PATCH 13/14] feat(subm): upsert 1 BLD requests into google sheet --- subm/package.json | 1 + subm/src/discordGuild.js | 36 +++++++++++++++++--- subm/src/request1bld.js | 72 +++++++++++++++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 14 deletions(-) diff --git a/subm/package.json b/subm/package.json index 5fae8b5..cbbf2c7 100644 --- a/subm/package.json +++ b/subm/package.json @@ -9,6 +9,7 @@ "lint:eslint": "eslint '**/*.js'", "lint:types": "tsc -p jsconfig.json", "start": "node src/subm.js", + "upsert": "node src/request1bld.js", "deploy": "gcloud app deploy", "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/subm/src/discordGuild.js b/subm/src/discordGuild.js index 5bc50a5..bccf703 100644 --- a/subm/src/discordGuild.js +++ b/subm/src/discordGuild.js @@ -35,7 +35,9 @@ function getContent(host, path, headers, { get }) { }); } -const query = opts => (opts ? `?${new URLSearchParams(opts).toString()}` : ''); +const { keys } = Object; +const query = opts => + keys(opts).length > 0 ? `?${new URLSearchParams(opts).toString()}` : ''; /** * Discord API (a small slice of it, anyway) @@ -101,6 +103,7 @@ function DiscordAPI(token, { get, setTimeout }) { * @returns {Promise} */ const getJSON = async path => { + console.log('getJSON', { path }); const body = await getContent(host, path, headers, { get }); const data = JSON.parse(body); // console.log('Discord done:', Object.keys(data)); @@ -112,14 +115,16 @@ function DiscordAPI(token, { get, setTimeout }) { }; return freeze({ + /** @param { Snowflake } channelID */ channels: channelID => { return freeze({ /** * @param {Record} opts * @returns {Promise} */ - getMessages: opts => + getMessages: (opts = {}) => getJSON(`${api}/channels/${channelID}/messages${query(opts)}`), + /** @param { Snowflake } messageID */ messages: messageID => freeze({ /** @@ -179,6 +184,27 @@ function avatar(user) { return `${avatarBase}/${user.id}/${user.avatar}.png`; } +/** + * @param {(opts: any) => Promise} fn + * @param {number} [limit] + * @template {{ id: string }} T + */ +async function paged(fn, limit = 100) { + /** @type {T[][]} */ + const pages = []; + /** @type { string | undefined } */ + let before; + do { + console.error('getting page', pages.length, before); + // eslint-disable-next-line no-await-in-loop + const page = await fn(before ? { limit, before } : { limit }); + if (!page.length) break; + before = page.slice(-1)[0].id; + pages.push(page); + } while (before); + return pages.flat(); +} + /** * @param {ReturnType['guilds']>} guild */ @@ -209,7 +235,7 @@ async function pagedMembers(guild) { * setTimeout: typeof setTimeout, * }} io */ -async function main(env, { stdout, get, setTimeout }) { +async function integrationTest(env, { stdout, get, setTimeout }) { const config = makeConfig(env); const discordAPI = DiscordAPI(config`DISCORD_API_TOKEN`, { get, setTimeout }); const guild = discordAPI.guilds(config`DISCORD_GUILD_ID`); @@ -223,7 +249,7 @@ async function main(env, { stdout, get, setTimeout }) { /* global require, process */ if (require.main === module) { - main(process.env, { + integrationTest(process.env, { stdout: process.stdout, // eslint-disable-next-line global-require get: require('https').get, @@ -234,4 +260,4 @@ if (require.main === module) { } /* global module */ -module.exports = { DiscordAPI, avatar, getContent }; +module.exports = { DiscordAPI, avatar, getContent, paged }; diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 4aac3bd..2fdfc94 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -1,7 +1,8 @@ /* eslint-disable no-await-in-loop */ // See https://github.com/Agoric/validator-profiles/wiki/Request-1-BLD -const { DiscordAPI, getContent } = require('./discordGuild'); +const { DiscordAPI, getContent, paged } = require('./discordGuild.js'); +const { upsert } = require('./sheetAccess'); const { searchBySender, transfers } = require('./tendermintRPC'); const config = { @@ -36,7 +37,7 @@ async function* authorizedRequests(channel, guild, role, quorum) { return detail; }; - const messages = await channel.getMessages({ limit: 100 }); + const messages = await paged(channel.getMessages); const hasAddr = messages.filter(msg => msg.content.match(/agoric1/)); if (!hasAddr) return; const hasChecks = hasAddr.filter(msg => { @@ -50,7 +51,7 @@ async function* authorizedRequests(channel, guild, role, quorum) { const endorsers = []; for (const endorsement of endorsements) { const detail = await getMemberDetail(endorsement.id); - if (detail.roles.includes(role)) { + if (detail && detail.roles && detail.roles.includes(role)) { endorsers.push(detail); } } @@ -102,6 +103,8 @@ async function requestStatus(channel, guild, roleID, { get }) { return result; } +const label = user => `${user.username}#${user.discriminator}`; + /** * @param {Record} env * @param {{ @@ -145,7 +148,6 @@ async function main(env, { get, setTimeout }) { env.REVIEWER_ROLE_ID, 2, )) { - const label = user => `${user.username}#${user.discriminator}`; const ok = endorsers.map(u => label(u.user)).join(' '); const hash = byRecipient.has(address) ? byRecipient.get(address).hash : ''; console.log( @@ -156,13 +158,65 @@ async function main(env, { get, setTimeout }) { } } +/** + * @param {string[]} args + * @param {Record} env + * @param {Object} io + * @param {typeof import('google-spreadsheet').GoogleSpreadsheet} io.GoogleSpreadsheet + * @param {typeof import('https').get} io.get + * @param {typeof setTimeout} io.setTimeout + */ +const main2 = async (args, env, { get, setTimeout, GoogleSpreadsheet }) => { + const discordAPI = DiscordAPI(env.DISCORD_API_TOKEN, { get, setTimeout }); + const guild = discordAPI.guilds(env.DISCORD_GUILD_ID); + + // to get mod-1-bld role id: + // console.log(await guild.roles()); + + const channel = discordAPI.channels(env.CHANNEL_ID); + + const creds = { + client_email: env.GOOGLE_SERVICES_EMAIL, + private_key: env.GCS_PRIVATE_KEY, + }; + // Initialize the sheet - doc ID is the long id in the sheets URL + const doc = new GoogleSpreadsheet(env.SHEET_1BLD_ID); + // Initialize Auth - see https://theoephraim.github.io/node-google-spreadsheet/#/getting-started/authentication + await doc.useServiceAccountAuth(creds); + await doc.loadInfo(); // loads document properties and worksheets + console.log(doc.title); + const sheet = doc.sheetsByIndex[0]; + + const memberLabel = mem => mem.nick || label(mem.user); + for await (const { message: msg, address, endorsers } of authorizedRequests( + channel, + guild, + env.REVIEWER_ROLE_ID, + 2, + )) { + upsert(sheet, address, { + Request: `https://discord.com/channels/585576150827532298/946137891023777802/${msg.id}`, + At: msg.timestamp.slice(0, '1999-01-01T12:59'.length).replace('T', ' '), + By: label(msg.author), + To: address, + Reviewers: endorsers.map(memberLabel).join(','), + }); + } +}; + /* global require, process, module */ if (require.main === module) { - main(process.env, { - // eslint-disable-next-line global-require - get: require('https').get, - setTimeout, - }).catch(err => console.error(err)); + main2( + process.argv.slice(2), + { ...process.env }, + { + // eslint-disable-next-line global-require + get: require('https').get, + setTimeout, + // eslint-disable-next-line global-require + GoogleSpreadsheet: require('google-spreadsheet').GoogleSpreadsheet, // please excuse CJS + }, + ).catch(err => console.error(err)); } module.exports = { requestStatus }; From b88d81529f677c2ad9be857578d4774ffc5d696f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 2 Apr 2022 01:20:59 -0500 Subject: [PATCH 14/14] feat(subm): include Tx in 1 BLD request sheet --- subm/src/request1bld.js | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/subm/src/request1bld.js b/subm/src/request1bld.js index 2fdfc94..a05d67a 100644 --- a/subm/src/request1bld.js +++ b/subm/src/request1bld.js @@ -85,16 +85,13 @@ async function requestStatus(channel, guild, roleID, { get }) { // console.log(byRecipient.keys()); const result = []; - for await (const { - message: { id, timestamp, author }, - address, - endorsers, - } of authorizedRequests(channel, guild, roleID, config.quorum)) { + const eachRequest = authorizedRequests(channel, guild, roleID, config.quorum); + for await (const { message, address, endorsers } of eachRequest) { const hash = byRecipient.has(address) ? byRecipient.get(address).hash : undefined; result.push({ - message: { id, timestamp, author }, + message, address, endorsers, hash, @@ -188,19 +185,20 @@ const main2 = async (args, env, { get, setTimeout, GoogleSpreadsheet }) => { const sheet = doc.sheetsByIndex[0]; const memberLabel = mem => mem.nick || label(mem.user); - for await (const { message: msg, address, endorsers } of authorizedRequests( - channel, - guild, - env.REVIEWER_ROLE_ID, - 2, - )) { - upsert(sheet, address, { - Request: `https://discord.com/channels/585576150827532298/946137891023777802/${msg.id}`, + const eachStatus = await requestStatus(channel, guild, env.REVIEWER_ROLE_ID, { + get, + }); + for (const { message: msg, address, endorsers, hash } of eachStatus) { + const record = { + To: address, + Link: `https://discord.com/channels/${env.DISCORD_GUILD_ID}/${env.CHANNEL_ID}/${msg.id}`, At: msg.timestamp.slice(0, '1999-01-01T12:59'.length).replace('T', ' '), By: label(msg.author), - To: address, Reviewers: endorsers.map(memberLabel).join(','), - }); + Tx: hash && `https://agoric.bigdipper.live/transactions/${hash}`, + Request: msg.content, + }; + await upsert(sheet, address, record); // do not interleave upserts! } };