diff --git a/src/backend/revoker/scanAttestations.ts b/src/backend/revoker/scanAttestations.ts index dad08dc09..ab668d6d1 100644 --- a/src/backend/revoker/scanAttestations.ts +++ b/src/backend/revoker/scanAttestations.ts @@ -33,14 +33,11 @@ function parseParams(event: ParsedEvent) { } function getDidUriFromAccountHex(didAccount: HexString) { - logger.debug('DID as HexString of Account Address: ' + didAccount); // SubScan returns some AttesterOf values as hex without the "0x" prefix // so we first parsed to a Uint8Array via `hexToU8a`, which can handle HexStrings with or without the prefix. const didU8a = hexToU8a(didAccount); - const didUri = Did.fromChain(didU8a as AccountId32); - logger.debug('Corresponding DID-URI: ' + didUri); - return didUri; + return Did.fromChain(didU8a as AccountId32); } export async function* scanAttestations() { diff --git a/src/backend/revoker/subScan.test.ts b/src/backend/revoker/subScan.test.ts index 45a2061d9..211b62aff 100644 --- a/src/backend/revoker/subScan.test.ts +++ b/src/backend/revoker/subScan.test.ts @@ -89,7 +89,7 @@ describe('subScan', () => { }); describe('subScanEventGenerator', () => { - it('should iterate through pages in reverse order', async () => { + it('should iterate through pages in ascending order', async () => { postResponse = { data: { count: 200, events: [] } }; const eventGenerator = subScanEventGenerator( @@ -110,13 +110,13 @@ describe('subScan', () => { // @ts-expect-error because TS infers wrong parameters expect(calls[0][1]).toMatchObject({ json: { page: 0, row: 1 } }); - // get last page + // get first page // @ts-expect-error because TS infers wrong parameters - expect(calls[2][1]).toMatchObject({ json: { page: 1, row: 100 } }); + expect(calls[2][1]).toMatchObject({ json: { page: 0, row: 100 } }); - // get first page + // get last page // @ts-expect-error because TS infers wrong parameters - expect(calls[4][1]).toMatchObject({ json: { page: 0, row: 100 } }); + expect(calls[4][1]).toMatchObject({ json: { page: 1, row: 100 } }); }); }); diff --git a/src/backend/revoker/subScan.ts b/src/backend/revoker/subScan.ts index 53e1b9fd9..2a2643355 100644 --- a/src/backend/revoker/subScan.ts +++ b/src/backend/revoker/subScan.ts @@ -10,7 +10,7 @@ const { subscan } = configuration; const SUBSCAN_MAX_ROWS = 100; const QUERY_INTERVAL_MS = 1000; -const BLOCK_RANGE_SIZE = 100_000; +const MAX_BLOCK_RANGE_SIZE = 100_000; const subscanAPI = `https://${subscan.network}.api.subscan.io`; const eventsListURL = `${subscanAPI}/api/v2/scan/events`; @@ -64,6 +64,7 @@ export interface EventsParamsJSON { export async function getEvents({ fromBlock, + toBlock, row = SUBSCAN_MAX_ROWS, eventId, ...parameters @@ -71,13 +72,14 @@ export async function getEvents({ module: string; // Pallet name eventId: string; // Event emitted fromBlock: number; + toBlock?: number; page: number; row?: number; }) { const payloadForEventsListRequest = { ...parameters, event_id: eventId, - block_range: `${fromBlock}-${fromBlock + BLOCK_RANGE_SIZE}`, + block_range: `${fromBlock}-${toBlock ?? fromBlock + MAX_BLOCK_RANGE_SIZE}`, order: 'asc', row, finalized: true, @@ -88,6 +90,12 @@ export async function getEvents({ JSON.stringify(payloadForEventsListRequest, null, 2), ); + if (parameters.page >= 100) { + throw new Error( + `Page ${parameters.page} exceeds Subscan's paging limit of 100.`, + ); + } + const { data: { count, events }, } = await got @@ -133,8 +141,6 @@ export async function getEvents({ }, ); - logger.debug('parsedEvents: ' + JSON.stringify(parsedEvents, null, 2)); - return { count, events: parsedEvents }; } @@ -163,46 +169,67 @@ export async function* subScanEventGenerator( for ( let fromBlock = startBlock; fromBlock < currentBlock; - fromBlock += BLOCK_RANGE_SIZE + fromBlock += MAX_BLOCK_RANGE_SIZE ) { - const parameters = { - module, - eventId, - fromBlock, - }; + // Subscan has a limit of 100 accessible pages for a given query. + // To stay within the limit and don't miss any events, we reduce the block range when necessary. + let rangeReducer = 1; + const endOfBigLoopBlock = fromBlock + MAX_BLOCK_RANGE_SIZE; + let nextFromBlock = fromBlock; + + while (nextFromBlock < endOfBigLoopBlock) { + const parameters = { + module, + eventId, + fromBlock: nextFromBlock, + toBlock: nextFromBlock + Math.ceil(MAX_BLOCK_RANGE_SIZE / rangeReducer), + }; - const { count } = await getEvents({ ...parameters, page: 0, row: 1 }); + const { count } = await getEvents({ ...parameters, page: 0, row: 1 }); - const blockRange = `${fromBlock} - ${fromBlock + BLOCK_RANGE_SIZE}`; + const blockRange = `${parameters.fromBlock} - ${parameters.toBlock}`; + + if (count === 0) { + logger.debug( + `No new "${eventId}" events found on SubScan in block range ${blockRange}.`, + ); + await sleep(QUERY_INTERVAL_MS); + nextFromBlock = parameters.toBlock; + continue; + } - if (count === 0) { logger.debug( - `No new "${eventId}" events found on SubScan in block range ${blockRange}.`, + `Found ${count} new "${eventId}" events on SubScan for in block range ${blockRange}.`, ); - await sleep(QUERY_INTERVAL_MS); - continue; - } - - logger.debug( - `Found ${count} new "${eventId}" events on SubScan for in block range ${blockRange}.`, - ); - const pages = Math.ceil(count / SUBSCAN_MAX_ROWS) - 1; + const pages = Math.ceil(count / SUBSCAN_MAX_ROWS) - 1; - for (let page = pages; page >= 0; page--) { - const { events } = await getEvents({ ...parameters, page }); - if (!events) { + if (pages > 100) { + rangeReducer += 1; + logger.debug( + `Reducing block range to comply with Subscan's page limit.`, + ); continue; } - logger.debug( - `Loaded page ${page} of "${eventId}" events in block range ${blockRange}.`, - ); - for (const event of await transform(events)) { - yield event; + for (let page = 0; page <= pages; page++) { + const { events } = await getEvents({ ...parameters, page }); + if (!events) { + continue; + } + + logger.debug( + `Loaded page ${page} of "${eventId}" events in block range ${blockRange}.`, + ); + for (const event of await transform(events)) { + yield event; + } + + await sleep(QUERY_INTERVAL_MS); } - await sleep(QUERY_INTERVAL_MS); + nextFromBlock = parameters.toBlock; + rangeReducer = 1; } } }