Skip to content

Commit

Permalink
Merge pull request #1381 from multiversx/API-72-first-integration-of-…
Browse files Browse the repository at this point in the history
…cs-in-api-service

API-72: first integration of chain simulator
  • Loading branch information
bogdan-rosianu authored Dec 3, 2024
2 parents d6887ed + 91290e3 commit 4b591c7
Show file tree
Hide file tree
Showing 30 changed files with 1,978 additions and 60 deletions.
59 changes: 59 additions & 0 deletions .github/workflows/chain-simulator-e2e-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
name: Chain simulator e2e tests workflow

on:
pull_request:

jobs:
test-chainsimulator-e2e:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [18.x]

steps:
- uses: actions/checkout@v3

- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
cache: 'npm'

- name: Build and start chain simulator
run: npm run start-chain-simulator

- name: Wait for services to be ready
run: |
echo "Waiting for services to be healthy..."
docker ps
docker logs chainsimulator
sleep 20 # Wait for 20 seconds to ensure services are up
- name: Print docker containers
run: docker ps

- name: Start API Docker containers
run: |
cd .
docker compose up -d
- name: Wait for API docker containers to start
run: |
sleep 20
docker ps
- run: npm ci
- run: npm run init

- name: Start API
run: |
npm run start:mainnet:e2e &
sleep 10 # Wait a little more to ensure the API is fully up
- name: Run e2e tests
run: npm run test:cs-e2e

- name: Stop API after tests
run: |
echo "Stopping the API..."
kill $(lsof -t -i:3001)
2 changes: 0 additions & 2 deletions .github/workflows/load-tests.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
name: Load Tests

on:
push:
branches: [main, development]
pull_request:
branches: [main, development]

Expand Down
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,7 @@ lerna-debug.log*

/src/plugins

.env
.env

# CS Environment variables
src/test/chain-simulator/config/.env
145 changes: 119 additions & 26 deletions config/config.e2e.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,51 @@ network: 'mainnet'
metaChainShardId: 4294967295
api:
public: true
publicPort: 3001
private: true
graphql: true
privatePort: 4001
websocket: true
cron:
cacheWarmer: true
fastWarm: false
queueWorker: true
elasticUpdater: false
flags:
useRequestCaching: true
useKeepAliveAgent: true
useTracing: false
useRequestLogging: false
useVmQueryTracing: false
processNfts: true
collectionPropertiesFromGateway: false
features:
eventsNotifier:
enabled: false
port: 5674
url: 'amqp://guest:guest@127.0.0.1:5672'
exchange: 'all_events'
queue: 'api-process-logs-and-events'
guestCaching:
enabled: false
hitsThreshold: 100
ttl: 12
transactionPool:
enabled: false
transactionPoolWarmer:
enabled: false
cronExpression: '*/5 * * * * *'
ttlInSeconds: 60
updateCollectionExtraDetails:
enabled: false
updateAccountExtraDetails:
enabled: false
transfersLast24hUrl: 'https://tools.multiversx.com/growth-api/explorer/widgets/most-used/applications'
marketplace:
enabled: false
serviceUrl: 'https://nfts-graph.multiversx.com/graphql'
exchange:
enabled: false
serviceUrl: 'https://graph.xexchange.com/graphql'
dataApi:
enabled: false
serviceUrl: 'https://data-api.multiversx.com'
Expand All @@ -13,35 +55,93 @@ features:
maxExpirySeconds: 86400
acceptedOrigins:
- ''
admins:
- ''
jwtSecret: ''
stakingV4:
enabled: false
cronExpression: '*/5 * * * * *'
activationEpoch: 1391
nodeEpochsLeft:
enabled: false
transactionProcessor:
enabled: false
maxLookBehind: 1000
transactionCompleted:
enabled: false
maxLookBehind: 1000
logLevel: 'Error'
transactionBatch:
enabled: false
maxLookBehind: 1000
deepHistory:
enabled: false
url: ''
cron:
transactionProcessor: false
transactionProcessorMaxLookBehind: 1000
cacheWarmer: false
flags:
useRequestCaching: true
useKeepAliveAgent: true
useTracing: false
collectionPropertiesFromGateway: false
statusChecker:
enabled: false
thresholds:
tokens: 1000
nodes: 5000
providers: 150
tokenSupplyCount: 100
tokenAssets: 100
tokenAccounts: 1000
tokenTransactions: 1000
nodeValidators: 3260
nftScamInfo:
enabled: false
processNfts:
enabled: false
nftQueueName: 'api-process-nfts'
deadLetterQueueName: 'api-process-nfts-dlq'
tps:
enabled: false
maxLookBehindNonces: 100
nodesFetch:
enabled: false
serviceUrl: 'https://api.multiversx.com'
tokensFetch:
enabled: false
serviceUrl: 'https://api.multiversx.com'
providersFetch:
enabled: false
serviceUrl: 'https://api.multiversx.com'
assetsFetch:
enabled: false
assetesUrl: 'https://tools.multiversx.com/assets-cdn'
image:
width: 600
height: 600
type: 'png'
aws:
s3KeyId: ''
s3Secret: ''
s3Bucket: 'media.elrond.com'
s3Region: ''
urls:
self: 'https://api.multiversx.com'
self: 'http://localhost:3001'
elastic:
- 'https://index.multiversx.com'
- 'http://localhost:9200'
gateway:
- 'https://gateway.multiversx.com'
- 'http://localhost:8085'
verifier: 'https://play-api.multiversx.com'
redis: '127.0.0.1'
rabbitmq: 'amqp://127.0.0.1:5672'
providers: 'https://internal-delegation-api.multiversx.com/providers'
providers: 'https://delegation-api.multiversx.com/providers'
delegation: 'https://delegation-api.multiversx.com'
media: 'https://media.elrond.com'
nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
tmp: '/tmp'
ipfs: 'https://ipfs.io/ipfs'
socket: 'socket-api-fra.multiversx.com'
maiarId: 'https://id-api.multiversx.com'
indexer:
type: 'elastic'
maxPagination: 10000
database:
enabled: false
url: 'mongodb://127.0.0.1:27017/api?authSource=admin'
type: 'mysql'
host: 'localhost'
port: 3306
username: 'root'
Expand All @@ -50,7 +150,8 @@ database:
caching:
cacheTtl: 6
processTtl: 600
poolLimit: 10
poolLimit: 50
cacheDuration: 3
keepAliveTimeout:
downstream: 61000
upstream: 60000
Expand All @@ -69,14 +170,6 @@ inflation:
- 1130177
- 924690
- 719203
security:
admins:
jwtSecret:
test:
mockKeybases: false
mockNodes: false
mockTokens: false
mockPath: './src/test/mocks/'
transaction-action:
mex:
microServiceUrl: 'https://graph.xexchange.com/graphql'
nftProcess:
parallelism: 1
maxRetries: 3
10 changes: 8 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
"start:mainnet": "npm run copy-mainnet-config & nest start",
"start:mainnet:watch": "npm run copy-mainnet-config & nest start --watch",
"start:mainnet:debug": "npm run copy-mainnet-config & nest start --watch --debug",
"start:mainnet:e2e": "npm run copy-e2e-mainnet-config & nest start",
"start:mainnet:e2e:watch": "npm run copy-e2e-mainnet-config & nest start --watch",
"start:mainnet:e2e:debug": "npm run copy-e2e-mainnet-config & nest start --watch --debug",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
"lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "npm run copy-mainnet-config & nest build & jest",
Expand All @@ -41,6 +44,7 @@
"test:e2e:local": "npm run test:e2e:warm & npm run test:e2e",
"test:api": "jest --config ./src/test/jest-api.json --runInBand --detectOpenHandles --forceExit",
"test:graph": "jest --config ./src/test/jest-graph-spec.json --runInBand --detectOpenHandles --forceExit",
"test:cs-e2e": "jest --config ./src/test/jest-chain-simulator.json --runInBand --detectOpenHandles --forceExit",
"init": "run-script-os",
"copy-mainnet-config": "run-script-os",
"copy-testnet-config": "run-script-os",
Expand Down Expand Up @@ -73,7 +77,9 @@
"copy-e2e-mainnet-config:nix": "cp ./config/config.e2e.mainnet.yaml ./config/config.yaml",
"copy-e2e-mocked-mainnet-config:nix": "cp ./config/config.e2e-mocked.mainnet.yaml ./config/config.yaml",
"copy-e2e-mainnet-config:windows": "copy .\\config\\config.e2e.mainnet.yaml .\\config\\config.yaml",
"copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml"
"copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml",
"start-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" up -d --build",
"stop-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" down"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.54.0",
Expand Down Expand Up @@ -209,4 +215,4 @@
"node_modules"
]
}
}
}
3 changes: 1 addition & 2 deletions src/common/assets/assets.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,7 @@ export class AssetsService {

const allAssets: { [key: string]: KeybaseIdentity } = {};
for (const asset of assets) {
const { identity, ...details } = asset;
allAssets[identity] = new KeybaseIdentity(details);
allAssets[asset.identity] = new KeybaseIdentity(asset);
}

return allAssets;
Expand Down
10 changes: 5 additions & 5 deletions src/common/keybase/keybase.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,12 @@ export class KeybaseService {

async confirmIdentityProfiles(): Promise<void> {
const identities = await this.getDistinctIdentities();
const keybaseIdentities = identities.map(identity => this.getProfile(identity));
const keybaseIdentities = await Promise.all(identities.map(identity => this.getProfile(identity)));
await this.cachingService.set(CacheInfo.IdentityProfilesKeybases.key, keybaseIdentities, CacheInfo.IdentityProfilesKeybases.ttl);
}

getProfile(identity: string): KeybaseIdentity | null {
const keybaseLocal = this.getProfileFromAssets(identity);
async getProfile(identity: string): Promise<KeybaseIdentity | null> {
const keybaseLocal = await this.getProfileFromAssets(identity);
if (keybaseLocal) {
this.logger.log(`Got profile details from assets for identity '${identity}'`);
return keybaseLocal;
Expand All @@ -116,8 +116,8 @@ export class KeybaseService {
return identityInfo;
}

getProfileFromAssets(identity: string): KeybaseIdentity | null {
const info = this.readIdentityInfo(identity);
async getProfileFromAssets(identity: string): Promise<KeybaseIdentity | null> {
const info = await this.readIdentityInfo(identity);
if (!info) {
return null;
}
Expand Down
Loading

0 comments on commit 4b591c7

Please sign in to comment.