fix: adapt to subscan paging limit

src/backend/revoker/scanAttestations.ts

@@ -33,14 +33,11 @@ function parseParams(event: ParsedEvent) {
 }
 
 function getDidUriFromAccountHex(didAccount: HexString) {
- logger.debug('DID as HexString of Account Address: ' + didAccount);
  // SubScan returns some AttesterOf values as hex without the "0x" prefix
  // so we first parsed to a Uint8Array via `hexToU8a`, which can handle HexStrings with or without the prefix.
  const didU8a = hexToU8a(didAccount);
 
- const didUri = Did.fromChain(didU8a as AccountId32);
- logger.debug('Corresponding DID-URI: ' + didUri);
- return didUri;
+ return Did.fromChain(didU8a as AccountId32);
 }
 
 export async function* scanAttestations() {

src/backend/revoker/subScan.test.ts

@@ -89,7 +89,7 @@ describe('subScan', () => {
  });
 
  describe('subScanEventGenerator', () => {
- it('should iterate through pages in reverse order', async () => {
+ it('should iterate through pages in ascending order', async () => {
  postResponse = { data: { count: 200, events: [] } };
 
  const eventGenerator = subScanEventGenerator(
@@ -110,13 +110,13 @@ describe('subScan', () => {
  // @ts-expect-error because TS infers wrong parameters
  expect(calls[0][1]).toMatchObject({ json: { page: 0, row: 1 } });
 
- // get last page
+ // get first page
  // @ts-expect-error because TS infers wrong parameters
- expect(calls[2][1]).toMatchObject({ json: { page: 1, row: 100 } });
+ expect(calls[2][1]).toMatchObject({ json: { page: 0, row: 100 } });
 
- // get first page
+ // get last page
  // @ts-expect-error because TS infers wrong parameters
- expect(calls[4][1]).toMatchObject({ json: { page: 0, row: 100 } });
+ expect(calls[4][1]).toMatchObject({ json: { page: 1, row: 100 } });
  });
  });
 

src/backend/revoker/subScan.ts

@@ -10,7 +10,7 @@ const { subscan } = configuration;
 
 const SUBSCAN_MAX_ROWS = 100;
 const QUERY_INTERVAL_MS = 1000;
-const BLOCK_RANGE_SIZE = 100_000;
+const MAX_BLOCK_RANGE_SIZE = 100_000;
 
 const subscanAPI = `https://${subscan.network}.api.subscan.io`;
 const eventsListURL = `${subscanAPI}/api/v2/scan/events`;
@@ -64,20 +64,22 @@ export interface EventsParamsJSON {
 
 export async function getEvents({
  fromBlock,
+ toBlock,
  row = SUBSCAN_MAX_ROWS,
  eventId,
  ...parameters
 }: {
  module: string; // Pallet name
  eventId: string; // Event emitted
  fromBlock: number;
+ toBlock?: number;
  page: number;
  row?: number;
 }) {
  const payloadForEventsListRequest = {
  ...parameters,
  event_id: eventId,
- block_range: `${fromBlock}-${fromBlock + BLOCK_RANGE_SIZE}`,
+ block_range: `${fromBlock}-${toBlock ?? fromBlock + MAX_BLOCK_RANGE_SIZE}`,
  order: 'asc',
  row,
  finalized: true,
@@ -88,6 +90,12 @@ export async function getEvents({
  JSON.stringify(payloadForEventsListRequest, null, 2),
  );
 
+ if (parameters.page >= 100) {
+ throw new Error(
+ `Page ${parameters.page} exceeds Subscan's paging limit of 100.`,
+ );
+ }
+
  const {
  data: { count, events },
  } = await got
@@ -133,8 +141,6 @@ export async function getEvents({
  },
  );
 
- logger.debug('parsedEvents: ' + JSON.stringify(parsedEvents, null, 2));
-
  return { count, events: parsedEvents };
 }
 
@@ -163,46 +169,67 @@ export async function* subScanEventGenerator(
  for (
  let fromBlock = startBlock;
  fromBlock < currentBlock;
- fromBlock += BLOCK_RANGE_SIZE
+ fromBlock += MAX_BLOCK_RANGE_SIZE
  ) {
- const parameters = {
- module,
- eventId,
- fromBlock,
- };
+ // Subscan has a limit of 100 accessible pages for a given query.
+ // To stay within the limit and don't miss any events, we reduce the block range when necessary.
+ let rangeReducer = 1;
+ const endOfBigLoopBlock = fromBlock + MAX_BLOCK_RANGE_SIZE;
+ let nextFromBlock = fromBlock;
+
+ while (nextFromBlock < endOfBigLoopBlock) {
+ const parameters = {
+ module,
+ eventId,
+ fromBlock: nextFromBlock,
+ toBlock: nextFromBlock + Math.ceil(MAX_BLOCK_RANGE_SIZE / rangeReducer),
+ };
 
- const { count } = await getEvents({ ...parameters, page: 0, row: 1 });
+  const { count } = await getEvents({ ...parameters, page: 0, row: 1 });
 
- const blockRange = `${fromBlock} - ${fromBlock + BLOCK_RANGE_SIZE}`;
+ const blockRange = `${parameters.fromBlock} - ${parameters.toBlock}`;
+
+ if (count === 0) {
+ logger.debug(
+ `No new "${eventId}" events found on SubScan in block range ${blockRange}.`,
+ );
+ await sleep(QUERY_INTERVAL_MS);
+ nextFromBlock = parameters.toBlock;
+ continue;
+ }
 
- if (count === 0) {
  logger.debug(
- `No new "${eventId}" events found on SubScan in block range ${blockRange}.`,
+ `Found ${count} new "${eventId}" events on SubScan for in block range ${blockRange}.`,
  );
- await sleep(QUERY_INTERVAL_MS);
- continue;
- }
-
- logger.debug(
- `Found ${count} new "${eventId}" events on SubScan for in block range ${blockRange}.`,
- );
 
- const pages = Math.ceil(count / SUBSCAN_MAX_ROWS) - 1;
+  const pages = Math.ceil(count / SUBSCAN_MAX_ROWS) - 1;
 
- for (let page = pages; page >= 0; page--) {
- const { events } = await getEvents({ ...parameters, page });
- if (!events) {
+ if (pages > 100) {
+ rangeReducer += 1;
+ logger.debug(
+ `Reducing block range to comply with Subscan's page limit.`,
+ );
  continue;
  }
 
- logger.debug(
- `Loaded page ${page} of "${eventId}" events in block range ${blockRange}.`,
- );
- for (const event of await transform(events)) {
- yield event;
+ for (let page = 0; page <= pages; page++) {
+ const { events } = await getEvents({ ...parameters, page });
+ if (!events) {
+ continue;
+ }
+
+ logger.debug(
+ `Loaded page ${page} of "${eventId}" events in block range ${blockRange}.`,
+ );
+ for (const event of await transform(events)) {
+ yield event;
+ }
+
+ await sleep(QUERY_INTERVAL_MS);
  }
 
- await sleep(QUERY_INTERVAL_MS);
+ nextFromBlock = parameters.toBlock;
+ rangeReducer = 1;
  }
  }
 }