API-72: first integration of chain simulator

.github/workflows/chain-simulator-e2e-tests.yml

@@ -0,0 +1,59 @@
+name: Chain simulator e2e tests workflow
+
+on:
+  pull_request:
+
+jobs:
+  test-chainsimulator-e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+
+      - name: Build and start chain simulator
+        run: npm run start-chain-simulator
+
+      - name: Wait for services to be ready
+        run: |
+          echo "Waiting for services to be healthy..."
+          docker ps
+          docker logs chainsimulator
+          sleep 20 # Wait for 20 seconds to ensure services are up
+
+      - name: Print docker containers
+        run: docker ps
+
+      - name: Start API Docker containers
+        run: |
+          cd .
+          docker compose up -d
+
+      - name: Wait for API docker containers to start
+        run: |
+          sleep 20
+          docker ps
+
+      - run: npm ci
+      - run: npm run init
+
+      - name: Start API
+        run: |
+          npm run start:mainnet:e2e &
+          sleep 10 # Wait a little more to ensure the API is fully up
+
+      - name: Run e2e tests
+        run: npm run test:cs-e2e
+
+      - name: Stop API after tests
+        run: |
+          echo "Stopping the API..."
+          kill $(lsof -t -i:3001)

.github/workflows/load-tests.yml

@@ -1,8 +1,6 @@
 name: Load Tests
 
 on:
-  push:
-    branches: [main, development]
   pull_request:
     branches: [main, development]
 

.gitignore

@@ -41,4 +41,7 @@ lerna-debug.log*
 
 /src/plugins
 
-.env
+.env
+
+# CS Environment variables
+src/test/chain-simulator/config/.env

config/config.e2e.mainnet.yaml

@@ -2,9 +2,51 @@ network: 'mainnet'
 metaChainShardId: 4294967295
 api:
   public: true
+  publicPort: 3001
   private: true
-  graphql: true
+  privatePort: 4001
+  websocket: true
+cron:
+  cacheWarmer: true
+  fastWarm: false
+  queueWorker: true
+  elasticUpdater: false
+flags:
+  useRequestCaching: true
+  useKeepAliveAgent: true
+  useTracing: false
+  useRequestLogging: false
+  useVmQueryTracing: false
+  processNfts: true
+  collectionPropertiesFromGateway: false
 features:
+  eventsNotifier:
+    enabled: false
+    port: 5674
+    url: 'amqp://guest:guest@127.0.0.1:5672'
+    exchange: 'all_events'
+    queue: 'api-process-logs-and-events'
+  guestCaching:
+    enabled: false
+    hitsThreshold: 100
+    ttl: 12
+  transactionPool:
+    enabled: false
+  transactionPoolWarmer:
+    enabled: false
+    cronExpression: '*/5 * * * * *'
+    ttlInSeconds: 60
+  updateCollectionExtraDetails:
+    enabled: false
+  updateAccountExtraDetails:
+    enabled: false
+    transfersLast24hUrl: 'https://tools.multiversx.com/growth-api/explorer/widgets/most-used/applications'
+  marketplace:
+    enabled: false
+    serviceUrl: 'https://nfts-graph.multiversx.com/graphql'
+  exchange:
+    enabled: false
+    serviceUrl: 'https://graph.xexchange.com/graphql'
   dataApi:
     enabled: false
     serviceUrl: 'https://data-api.multiversx.com'
@@ -13,35 +55,93 @@ features:
     maxExpirySeconds: 86400
     acceptedOrigins:
       - ''
+    admins:
+      - ''
+    jwtSecret: ''
+  stakingV4:
+    enabled: false
+    cronExpression: '*/5 * * * * *'
+    activationEpoch: 1391
+  nodeEpochsLeft:
+    enabled: false
+  transactionProcessor:
+    enabled: false
+    maxLookBehind: 1000
+  transactionCompleted:
+    enabled: false
+    maxLookBehind: 1000
+    logLevel: 'Error'
+  transactionBatch:
+    enabled: false
+    maxLookBehind: 1000
   deepHistory:
     enabled: false
     url: ''
-cron:
-  transactionProcessor: false
-  transactionProcessorMaxLookBehind: 1000
-  cacheWarmer: false
-flags:
-  useRequestCaching: true
-  useKeepAliveAgent: true
-  useTracing: false
-  collectionPropertiesFromGateway: false
+  statusChecker:
+    enabled: false
+    thresholds:
+      tokens: 1000
+      nodes: 5000
+      providers: 150
+      tokenSupplyCount: 100
+      tokenAssets: 100
+      tokenAccounts: 1000
+      tokenTransactions: 1000
+      nodeValidators: 3260
+  nftScamInfo:
+    enabled: false
+  processNfts:
+    enabled: false
+    nftQueueName: 'api-process-nfts'
+    deadLetterQueueName: 'api-process-nfts-dlq'
+  tps:
+    enabled: false
+    maxLookBehindNonces: 100
+  nodesFetch:
+    enabled: false
+    serviceUrl: 'https://api.multiversx.com'
+  tokensFetch:
+    enabled: false
+    serviceUrl: 'https://api.multiversx.com'
+  providersFetch:
+    enabled: false
+    serviceUrl: 'https://api.multiversx.com'
+  assetsFetch:
+    enabled: false
+    assetesUrl: 'https://tools.multiversx.com/assets-cdn'
+image:
+  width: 600
+  height: 600
+  type: 'png'
+aws:
+  s3KeyId: ''
+  s3Secret: ''
+  s3Bucket: 'media.elrond.com'
+  s3Region: ''
 urls:
-  self: 'https://api.multiversx.com'
+  self: 'http://localhost:3001'
   elastic:
-    - 'https://index.multiversx.com'
+    - 'http://localhost:9200'
   gateway:
-    - 'https://gateway.multiversx.com'
+    - 'http://localhost:8085'
   verifier: 'https://play-api.multiversx.com'
   redis: '127.0.0.1'
   rabbitmq: 'amqp://127.0.0.1:5672'
-  providers: 'https://internal-delegation-api.multiversx.com/providers'
+  providers: 'https://delegation-api.multiversx.com/providers'
   delegation: 'https://delegation-api.multiversx.com'
   media: 'https://media.elrond.com'
   nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
+  tmp: '/tmp'
+  ipfs: 'https://ipfs.io/ipfs'
+  socket: 'socket-api-fra.multiversx.com'
   maiarId: 'https://id-api.multiversx.com'
+indexer:
+  type: 'elastic'
+  maxPagination: 10000
 database:
   enabled: false
   url: 'mongodb://127.0.0.1:27017/api?authSource=admin'
+  type: 'mysql'
   host: 'localhost'
   port: 3306
   username: 'root'
@@ -50,7 +150,8 @@ database:
 caching:
   cacheTtl: 6
   processTtl: 600
-  poolLimit: 10
+  poolLimit: 50
+  cacheDuration: 3
 keepAliveTimeout:
   downstream: 61000
   upstream: 60000
@@ -69,14 +170,6 @@ inflation:
   - 1130177
   - 924690
   - 719203
-security:
-  admins:
-  jwtSecret:
-test:
-  mockKeybases: false
-  mockNodes: false
-  mockTokens: false
-  mockPath: './src/test/mocks/'
-transaction-action:
-  mex:
-    microServiceUrl: 'https://graph.xexchange.com/graphql'
+nftProcess:
+  parallelism: 1
+  maxRetries: 3

package.json

@@ -28,6 +28,9 @@
     "start:mainnet": "npm run copy-mainnet-config & nest start",
     "start:mainnet:watch": "npm run copy-mainnet-config & nest start --watch",
     "start:mainnet:debug": "npm run copy-mainnet-config & nest start --watch --debug",
+    "start:mainnet:e2e": "npm run copy-e2e-mainnet-config & nest start",
+    "start:mainnet:e2e:watch": "npm run copy-e2e-mainnet-config & nest start --watch",
+    "start:mainnet:e2e:debug": "npm run copy-e2e-mainnet-config & nest start --watch --debug",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
     "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "npm run copy-mainnet-config & nest build & jest",
@@ -41,6 +44,7 @@
     "test:e2e:local": "npm run test:e2e:warm & npm run test:e2e",
     "test:api": "jest --config ./src/test/jest-api.json --runInBand --detectOpenHandles --forceExit",
     "test:graph": "jest --config ./src/test/jest-graph-spec.json --runInBand --detectOpenHandles --forceExit",
+    "test:cs-e2e": "jest --config ./src/test/jest-chain-simulator.json --runInBand --detectOpenHandles --forceExit",
     "init": "run-script-os",
     "copy-mainnet-config": "run-script-os",
     "copy-testnet-config": "run-script-os",
@@ -73,7 +77,9 @@
     "copy-e2e-mainnet-config:nix": "cp ./config/config.e2e.mainnet.yaml ./config/config.yaml",
     "copy-e2e-mocked-mainnet-config:nix": "cp ./config/config.e2e-mocked.mainnet.yaml ./config/config.yaml",
     "copy-e2e-mainnet-config:windows": "copy .\\config\\config.e2e.mainnet.yaml .\\config\\config.yaml",
-    "copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml"
+    "copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml",
+    "start-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" up -d --build",
+    "stop-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" down"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.54.0",
@@ -209,4 +215,4 @@
       "node_modules"
     ]
   }
-}
+}

src/common/assets/assets.service.ts

@@ -202,8 +202,7 @@ export class AssetsService {
 
     const allAssets: { [key: string]: KeybaseIdentity } = {};
     for (const asset of assets) {
-      const { identity, ...details } = asset;
-      allAssets[identity] = new KeybaseIdentity(details);
+      allAssets[asset.identity] = new KeybaseIdentity(asset);
     }
 
     return allAssets;

src/common/keybase/keybase.service.ts

@@ -97,12 +97,12 @@ export class KeybaseService {
 
   async confirmIdentityProfiles(): Promise<void> {
     const identities = await this.getDistinctIdentities();
-    const keybaseIdentities = identities.map(identity => this.getProfile(identity));
+    const keybaseIdentities = await Promise.all(identities.map(identity => this.getProfile(identity)));
     await this.cachingService.set(CacheInfo.IdentityProfilesKeybases.key, keybaseIdentities, CacheInfo.IdentityProfilesKeybases.ttl);
   }
 
-  getProfile(identity: string): KeybaseIdentity | null {
-    const keybaseLocal = this.getProfileFromAssets(identity);
+  async getProfile(identity: string): Promise<KeybaseIdentity | null> {
+    const keybaseLocal = await this.getProfileFromAssets(identity);
     if (keybaseLocal) {
       this.logger.log(`Got profile details from assets for identity '${identity}'`);
       return keybaseLocal;
@@ -116,8 +116,8 @@ export class KeybaseService {
     return identityInfo;
   }
 
-  getProfileFromAssets(identity: string): KeybaseIdentity | null {
-    const info = this.readIdentityInfo(identity);
+  async getProfileFromAssets(identity: string): Promise<KeybaseIdentity | null> {
+    const info = await this.readIdentityInfo(identity);
     if (!info) {
       return null;
     }