Nodes and tokens fetch from external api

config/config.devnet.yaml

@@ -94,6 +94,12 @@ features:
   tps:
     enabled: true
     maxLookBehindNonces: 100
+  nodesFetch:
+    enabled: true
+    serviceUrl: 'https://devnet-api.multiversx.com'
+  tokensFetch:
+    enabled: true
+    serviceUrl: 'https://devnet-api.multiversx.com'
 image:
   width: 600
   height: 600

config/config.mainnet.yaml

@@ -97,7 +97,13 @@ features:
     deadLetterQueueName: 'api-process-nfts-dlq'
   tps:
     enabled: true
-    maxLookBehindNonces: 100  
+    maxLookBehindNonces: 100
+  nodesFetch:
+    enabled: true
+    serviceUrl: 'https://api.multiversx.com'
+  tokensFetch:
+    enabled: true
+    serviceUrl: 'https://api.multiversx.com'
 image:
   width: 600
   height: 600

config/config.testnet.yaml

@@ -97,6 +97,12 @@ features:
   tps:
     enabled: true
     maxLookBehindNonces: 100
+  nodesFetch:
+    enabled: true
+    serviceUrl: 'https://testnet-api.multiversx.com'
+  tokensFetch:
+    enabled: true
+    serviceUrl: 'https://testnet-api.multiversx.com'
 image:
   width: 600
   height: 600

docker-compose.yml

@@ -15,7 +15,7 @@ services:
     environment:
       - MYSQL_ROOT_PASSWORD=root
       - MYSQL_DATABASE=api
-  
+
   mongodbdatabase:
     image: mongo:latest
     environment:

src/common/api-config/api.config.service.ts

@@ -869,4 +869,30 @@ export class ApiConfigService {
 
     return deepHistoryUrl;
   }
+
+  isTokensFetchFeatureEnabled(): boolean {
+    return this.configService.get<boolean>('features.tokensFetch.enabled') ?? false;
+  }
+
+  getTokensFetchServiceUrl(): string {
+    const serviceUrl = this.configService.get<string>('features.tokensFetch.serviceUrl');
+    if (!serviceUrl) {
+      throw new Error('No tokens fetch service url present');
+    }
+
+    return serviceUrl;
+  }
+
+  isNodesFetchFeatureEnabled(): boolean {
+    return this.configService.get<boolean>('features.nodesFetch.enabled') ?? false;
+  }
+
+  getNodesFetchServiceUrl(): string {
+    const serviceUrl = this.configService.get<string>('features.nodesFetch.serviceUrl');
+    if (!serviceUrl) {
+      throw new Error('No nodes fetch service url present');
+    }
+
+    return serviceUrl;
+  }
 }

src/endpoints/nodes/node.service.ts

@@ -25,6 +25,7 @@
 import { NodeAuctionFilter } from "./entities/node.auction.filter";
 import { Identity } from "../identities/entities/identity";
 import { NodeSortAuction } from "./entities/node.sort.auction";
+import { ApiService } from "@multiversx/sdk-nestjs-http";
 
 @Injectable()
 export class NodeService {
@@ -42,7 +43,8 @@
     private readonly protocolService: ProtocolService,
     private readonly keysService: KeysService,
     @Inject(forwardRef(() => IdentitiesService))
-    private readonly identitiesService: IdentitiesService
+    private readonly identitiesService: IdentitiesService,
+    private readonly apiService: ApiService,
   ) { }
 
   private getIssues(node: Node, version: string | undefined): string[] {
@@ -372,6 +374,10 @@
   }
 
   async getAllNodesRaw(): Promise<Node[]> {
+    if (this.apiConfigService.isNodesFetchFeatureEnabled()) {
+      return await this.getAllNodesFromApi();
+    }
+
     const nodes = await this.getHeartbeatValidatorsAndQueue();
 
     await this.applyNodeIdentities(nodes);
@@ -392,6 +398,35 @@
     return nodes;
   }
 
+  private async getAllNodesFromApi(): Promise<Node[]> {
+    try {
+      let from = 0;
+      const size = 10000;
+      const nodes = [];
+      let currentNodes = [];
+
+      do {
+        const { data } = await this.apiService.get(`${this.apiConfigService.getNodesFetchServiceUrl()}/nodes`, {
+          params: {
+            from,
+            size
+          }
+        });
+
+        nodes.push(...data);
+        currentNodes = data
+        from += size;
+      } while (currentNodes.length === size);
+
+      return nodes;
+    } catch (error) {
+      this.logger.error('An unhandled error occurred when getting nodes from API');
+      this.logger.error(error);
+
+      throw error;
+    }
+  }
+
   async processAuctions(nodes: Node[], auctions: Auction[]) {
     const minimumAuctionStake = await this.stakeService.getMinimumAuctionStake(auctions);
     const dangerZoneThreshold = BigInt(minimumAuctionStake) * BigInt(105) / BigInt(100);

src/endpoints/tokens/token.service.ts

@@ -20,7 +20,7 @@
 import { TokenWithRoles } from "./entities/token.with.roles";
 import { TokenWithRolesFilter } from "./entities/token.with.roles.filter";
 import { AddressUtils, BinaryUtils, NumberUtils, TokenUtils } from "@multiversx/sdk-nestjs-common";
-import { ApiUtils } from "@multiversx/sdk-nestjs-http";
+import { ApiService, ApiUtils } from "@multiversx/sdk-nestjs-http";
 import { CacheService } from "@multiversx/sdk-nestjs-cache";
 import { IndexerService } from "src/common/indexer/indexer.service";
 import { OriginLogger } from "@multiversx/sdk-nestjs-common";
@@ -63,6 +63,7 @@
     private readonly collectionService: CollectionService,
     private readonly dataApiService: DataApiService,
     private readonly mexPairService: MexPairService,
+    private readonly apiService: ApiService,
   ) { }
 
   async isToken(identifier: string): Promise<boolean> {
@@ -719,8 +720,11 @@
   }
 
   async getAllTokensRaw(): Promise<TokenDetailed[]> {
-    this.logger.log(`Starting to fetch all tokens`);
+    if (this.apiConfigService.isTokensFetchFeatureEnabled()) {
+      return await this.getAllTokensFromApi();
+    }
 
+    this.logger.log(`Starting to fetch all tokens`);
     const tokensProperties = await this.esdtService.getAllFungibleTokenProperties();
     let tokens = tokensProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties));
 
@@ -856,6 +860,35 @@
     );
   }
 
+  private async getAllTokensFromApi(): Promise<TokenDetailed[]> {
+    try {
+      let from = 0;
+      const size = 10000;
+      const tokens = [];
+      let currentTokens = [];
+
+      do {
+        const { data } = await this.apiService.get(`${this.apiConfigService.getTokensFetchServiceUrl()}/tokens`, {
+          params: {
+            from,
+            size
+          }
+        });
+
+        tokens.push(...data);
+        currentTokens = data;
+        from += size;
+      } while (currentTokens.length === size);
+
+      return tokens;
+    } catch (error) {
+      this.logger.error('An unhandled error occurred when getting tokens from API');
+      this.logger.error(error);
+
+      throw error;
+    }
+  }
+
   private async getTotalTransactions(token: TokenDetailed): Promise<{ count: number, lastUpdatedAt: number } | undefined> {
     try {
       const count = await this.transactionService.getTransactionCount(new TransactionFilter({ tokens: [token.identifier, ...token.assets?.extraTokens ?? []] }));

src/test/unit/services/nodes.spec.ts

@@ -19,6 +19,7 @@ import { IdentitiesService } from "src/endpoints/identities/identities.service";
 import { NodeAuctionFilter } from "src/endpoints/nodes/entities/node.auction.filter";
 import * as fs from 'fs';
 import * as path from 'path';
+import { ApiService } from "@multiversx/sdk-nestjs-http";
 
 describe('NodeService', () => {
   let nodeService: NodeService;
@@ -27,6 +28,7 @@ describe('NodeService', () => {
   let apiConfigService: ApiConfigService;
   let gatewayService: GatewayService;
   let identitiesService: IdentitiesService;
+  let apiService: ApiService;
 
   beforeAll(async () => {
     const moduleRef = await Test.createTestingModule({
@@ -58,6 +60,8 @@ describe('NodeService', () => {
             getAuctionContractAddress: jest.fn(),
             isNodeSyncProgressEnabled: jest.fn(),
             isNodeEpochsLeftEnabled: jest.fn(),
+            isNodesFetchFeatureEnabled: jest.fn(),
+            getNodesFetchServiceUrl: jest.fn(),
           },
         },
         {
@@ -108,6 +112,12 @@ describe('NodeService', () => {
             getAllIdentities: jest.fn(),
           },
         },
+        {
+          provide: ApiService,
+          useValue: {
+            get: jest.fn(),
+          },
+        },
       ],
     }).compile();
 
@@ -117,6 +127,7 @@ describe('NodeService', () => {
     apiConfigService = moduleRef.get<ApiConfigService>(ApiConfigService);
     gatewayService = moduleRef.get<GatewayService>(GatewayService);
     identitiesService = moduleRef.get<IdentitiesService>(IdentitiesService);
+    apiService = moduleRef.get<ApiService>(ApiService);
   });
 
   beforeEach(() => { jest.restoreAllMocks(); });
@@ -427,6 +438,22 @@ describe('NodeService', () => {
       });
     });
 
+    describe('getAllNodes', () => {
+      it('should return values from external api', async () => {
+        const mockNodes = JSON.parse(fs.readFileSync(path.join(__dirname, '../../mocks/nodes.mock.json'), 'utf-8'));
+        nodeService['cacheService'].getOrSet = jest.fn().mockImplementation((_, callback) => callback());
+        jest.spyOn(apiConfigService, 'isNodesFetchFeatureEnabled').mockReturnValue(true);
+        jest.spyOn(apiConfigService, 'getNodesFetchServiceUrl').mockReturnValue('https://testnet-api.multiversx.com');
+        jest.spyOn(apiService, 'get').mockResolvedValueOnce({data: mockNodes.length}).mockResolvedValueOnce({data: mockNodes});
+        const getHeartbeatValidatorsAndQueueSpy = jest.spyOn(nodeService, 'getHeartbeatValidatorsAndQueue');
+
+        const result = await nodeService.getAllNodes();
+        expect(result).toEqual(mockNodes);
+        expect(apiService.get).toHaveBeenCalledTimes(2);
+        expect(getHeartbeatValidatorsAndQueueSpy).not.toHaveBeenCalled();
+      });
+    });
+
     describe('deleteOwnersForAddressInCache', () => {
       it('should return an empty array if no cache entries are found for an address', async () => {
         const address = 'erd1qqqqqqqqqqqqqpgqp699jngundfqw07d8jzkepucvpzush6k3wvqyc44rx';

src/test/unit/services/tokens.spec.ts

@@ -25,13 +25,16 @@ import { TransferService } from "src/endpoints/transfers/transfer.service";
 import { MexPairService } from "src/endpoints/mex/mex.pair.service";
 import * as fs from 'fs';
 import * as path from 'path';
+import { ApiService } from "@multiversx/sdk-nestjs-http";
 
 describe('Token Service', () => {
   let tokenService: TokenService;
   let esdtService: EsdtService;
   let collectionService: CollectionService;
   let indexerService: IndexerService;
   let assetsService: AssetsService;
+  let apiService: ApiService;
+  let apiConfigService: ApiConfigService;
 
   beforeEach(async () => {
     const moduleRef = await Test.createTestingModule({
@@ -85,6 +88,8 @@ describe('Token Service', () => {
           provide: ApiConfigService,
           useValue: {
             getIsIndexerV3FlagActive: jest.fn(),
+            isTokensFetchFeatureEnabled: jest.fn(),
+            getTokensFetchServiceUrl: jest.fn(),
           },
         },
         {
@@ -131,6 +136,12 @@ describe('Token Service', () => {
             getAllMexPairs: jest.fn(),
           },
         },
+        {
+          provide: ApiService,
+          useValue: {
+            get: jest.fn(),
+          },
+        },
       ],
     }).compile();
 
@@ -139,6 +150,8 @@ describe('Token Service', () => {
     collectionService = moduleRef.get<CollectionService>(CollectionService);
     indexerService = moduleRef.get<IndexerService>(IndexerService);
     assetsService = moduleRef.get<AssetsService>(AssetsService);
+    apiService = moduleRef.get<ApiService>(ApiService);
+    apiConfigService = moduleRef.get<ApiConfigService>(ApiConfigService);
   });
 
   afterEach(() => {
@@ -605,6 +618,19 @@ describe('Token Service', () => {
       },
     ];
 
+    it('should return values from external api', async () => {
+      tokenService['cachingService'].getOrSet = jest.fn().mockImplementation((_, callback) => callback());
+      jest.spyOn(apiConfigService, 'isTokensFetchFeatureEnabled').mockReturnValue(true);
+      jest.spyOn(apiConfigService, 'getTokensFetchServiceUrl').mockReturnValue('https://testnet-api.multiversx.com');
+      jest.spyOn(apiService, 'get').mockResolvedValueOnce({data: mockTokens.length}).mockResolvedValueOnce({data: mockTokens});
+
+      const result = await tokenService.getAllTokens();
+      expect(result).toEqual(mockTokens);
+      expect(apiService.get).toHaveBeenCalledTimes(2);
+      expect(esdtService.getAllFungibleTokenProperties).not.toHaveBeenCalled();
+      expect(collectionService.getNftCollections).not.toHaveBeenCalled();
+    });
+
     it('should return values from cache', async () => {
       const cachedValueMock = jest.spyOn(tokenService['cachingService'], 'getOrSet').mockResolvedValue(mockTokens);