Merge pull request #18 from dappforce/feat/improve-squid-api-usage

Feat: improve squid api usage
dappforce · Aug 30, 2023 · 8c1be92 · 8c1be92
2 parents b0de57e + 8aae49b
commit 8c1be92
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 23 deletions.
diff --git a/deployment/staging/all.yaml b/deployment/staging/all.yaml
@@ -61,8 +61,8 @@ data:
   IPFS_ADMIN_NODE_URL: "https://gw.crustfiles.app/"
   BOT_PORT: "3000"
   API_NO_ADMIN_PROTECTION_STR: "false"
-  DATA_PROVIDER_SQUID_WS_URL: "wss://squid.subsquid.io/xsocial-wss/graphql"
-  DATA_PROVIDER_SQUID_HTTPS_URL: "https://squid.subsquid.io/xsocial/graphql"
+  DATA_PROVIDER_SQUID_WS_URL: "wss://squid.subsquid.io/xsocial/v/v345/graphql"
+  DATA_PROVIDER_SQUID_HTTPS_URL: "https://squid.subsquid.io/xsocial/v/v345/graphql"
   TELEGRAM_BOT_GRILL_REDIRECTION_HREF: "https://grill.chat"
   TELEGRAM_TEMPORARY_LINKING_ID_EXPIRATION_TIME_MINS_STR: "10"
   FIREBASE_ADMIN_SDK_CREDS: "/var/run/secrets/firebase.json"

diff --git a/src/modules/dataProviders/dto/squid/squidResponse.dto.ts b/src/modules/dataProviders/dto/squid/squidResponse.dto.ts
@@ -90,6 +90,13 @@ export class SquidSubscriptionNotificationsResponseDto {
   };
 }
 
+export class SquidSubscriptionBatchNotificationsResponseDto {
+  id: string;
+  batchStartBlockNumber: string;
+  batchEndBlockNumber: string;
+  activityIds: string[];
+}
+
 export class SquidSubscriptionsActivitiesResponseDto {
   id: string;
   blockNumber: string;

diff --git a/src/modules/dataProviders/providers/squid/queries.ts b/src/modules/dataProviders/providers/squid/queries.ts
@@ -9,6 +9,16 @@ export const squidSubQueryNotificationsShort = `
         }
     `;
 
+export const squidSubQueryBatchNotifications = `
+        subscription {
+          inBatchNotifications(limit: 1, orderBy: batchEndBlockNumber_DESC) {
+            batchStartBlockNumber
+            batchEndBlockNumber
+            activityIds
+          }
+        }
+    `;
+
 export const squidSubQueryActivitiesShort = `
         subscription {
           activities(limit: 500, orderBy: activity_blockNumber_DESC) {

diff --git a/src/modules/dataProviders/providers/squid/squidSubscription.provider.ts b/src/modules/dataProviders/providers/squid/squidSubscription.provider.ts
@@ -6,15 +6,17 @@ import {
   SquidActivitiesResponseDto,
   SquidNotificationsResponseDto,
   SquidSubscriptionNotificationsResponseDto,
-  SquidSubscriptionsActivitiesResponseDto
+  SquidSubscriptionsActivitiesResponseDto,
+  SquidSubscriptionBatchNotificationsResponseDto
 } from '../../dto/squid/squidResponse.dto';
 import { SquidApiQueryName } from '../../typeorm/squidDataSubscriptionStatus';
 import { DataProvidersService } from '../../services/dataProviders.service';
 import { EventName } from '../../dto/squid/squidEvents.dto';
 import { newLogger } from '@subsocial/utils';
 import {
   squidSubQueryNotificationsShort,
-  getSquidQueryNotificationsFull
+  getSquidQueryNotificationsFull,
+  squidSubQueryBatchNotifications
 } from './queries';
 import { SquidHelper } from './squid.helper';
 
@@ -35,38 +37,39 @@ export class SquidSubscriptionDataProvider implements OnApplicationBootstrap {
   subscribeToNotifications() {
     this.graphqlWsClient.subscribe(
       {
-        query: squidSubQueryNotificationsShort
+        query: squidSubQueryBatchNotifications
       },
       {
         next: async (data) => {
           this.logger.info(`New squid status:`);
+
+          this.logger.info('RAW subscription data :: data >>> ');
+          console.dir(data.data, { depth: null });
+
           const notProcessedSubData = (await this.filterSubscriptionData(
-            SquidApiQueryName.notifications,
-            <Array<SquidSubscriptionNotificationsResponseDto>>(
-              data.data.notifications
+            SquidApiQueryName.inBatchNotifications,
+            <Array<SquidSubscriptionBatchNotificationsResponseDto>>(
+              data.data.inBatchNotifications
             )
-          )) as Array<SquidSubscriptionNotificationsResponseDto>;
+          )) as Array<SquidSubscriptionBatchNotificationsResponseDto>;
 
-          this.logger.info(
-            'RAW subscription data :: length - ',
-            (<Array<SquidNotificationsResponseDto>>data.data.notifications)
-              .length
-          );
-          this.logger.info(
-            'filtered subscription data by blockNumber :: length - ',
+          console.log(
+            'notProcessedSubData.length - ',
             notProcessedSubData.length
           );
 
           if (notProcessedSubData.length === 0) return;
 
+          const mergedNotificationIds = notProcessedSubData
+            .map((batchData) => batchData.activityIds)
+            .flat();
+
+          console.log('mergedNotificationIds - ', mergedNotificationIds);
+
           const fullData = await this.squidHelper.runSquidApiQuery<
             SquidApiQueryName.notifications,
             SquidNotificationsResponseDto
-          >(
-            getSquidQueryNotificationsFull(
-              notProcessedSubData.map((item) => item.id)
-            )
-          );
+          >(getSquidQueryNotificationsFull(mergedNotificationIds));
 
           const notProcessedSubDataWithoutWrappers =
             this.filterSubDataNotificationsByContentExtensionWrappers(
@@ -84,7 +87,7 @@ export class SquidSubscriptionDataProvider implements OnApplicationBootstrap {
             notProcessedSubDataWithoutWrappers
           );
           await this.dataProvidersService.updateStatusByQueryName({
-            name: SquidApiQueryName.notifications,
+            name: SquidApiQueryName.inBatchNotifications,
             lastProcessedBlockNumber: Number.parseInt(
               notProcessedSubDataWithoutWrappers[0].activity.blockNumber
             )
@@ -114,6 +117,7 @@ export class SquidSubscriptionDataProvider implements OnApplicationBootstrap {
     subscriptionEntitiesList: Array<
       | SquidSubscriptionNotificationsResponseDto
       | SquidSubscriptionsActivitiesResponseDto
+      | SquidSubscriptionBatchNotificationsResponseDto
     >
   ) {
     const subscriptionStatusData =
@@ -131,6 +135,7 @@ export class SquidSubscriptionDataProvider implements OnApplicationBootstrap {
             subscriptionStatusData.lastProcessedBlockNumber
         );
         break;
+
       case SquidApiQueryName.notifications:
         return (<Array<SquidNotificationsResponseDto>>(
           subscriptionEntitiesList
@@ -140,6 +145,15 @@ export class SquidSubscriptionDataProvider implements OnApplicationBootstrap {
             subscriptionStatusData.lastProcessedBlockNumber
         );
         break;
+      case SquidApiQueryName.inBatchNotifications:
+        return (<Array<SquidSubscriptionBatchNotificationsResponseDto>>(
+          subscriptionEntitiesList
+        )).filter(
+          (inBatchNotifications) =>
+            Number.parseInt(inBatchNotifications.batchStartBlockNumber) >
+            subscriptionStatusData.lastProcessedBlockNumber
+        );
+        break;
       default:
     }
   }

diff --git a/src/modules/dataProviders/typeorm/squidDataSubscriptionStatus.ts b/src/modules/dataProviders/typeorm/squidDataSubscriptionStatus.ts
@@ -2,7 +2,8 @@ import { Entity, PrimaryColumn, Column, ObjectIdColumn } from 'typeorm';
 
 export enum SquidApiQueryName {
   activities = 'activities',
-  notifications = 'notifications'
+  notifications = 'notifications',
+  inBatchNotifications = 'inBatchNotifications'
 }
 
 export enum AppEnvironment {