Skip to content

Commit

Permalink
feat: optimize query onchain evm_transaction, evm_internal_transactio…
Browse files Browse the repository at this point in the history 
…n by viem
  • Loading branch information
@fibonacci998
fibonacci998 committed Sep 4, 2024
1 parent 1594bde commit e007520
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 56 deletions.
30 changes: 14 additions & 16 deletions src/services/evm/crawl_evm_transaction.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ export default class CrawlEvmTransactionService extends BullableService {
})
.flat();
const receiptTxs = await this.getListTxReceipt(blocks);
const receiptTxsByHash = _.keyBy(receiptTxs, 'transactionHash');
if (receiptTxs.find((tx) => tx == null)) {
throw Error('Found null transaction receipt');
}
Expand All @@ -123,10 +124,7 @@ export default class CrawlEvmTransactionService extends BullableService {
throw Error('Transaction count not match');
}
offchainTxs.forEach((offchainTx) => {
const receiptTx = receiptTxs.find(
(tx) =>
tx && tx.transactionHash && tx.transactionHash === offchainTx.hash
);
const receiptTx = receiptTxsByHash[offchainTx.hash];
if (!receiptTx) {
throw Error('Transaction receipt not found');
}
Expand All @@ -135,9 +133,9 @@ export default class CrawlEvmTransactionService extends BullableService {
...receiptTx.logs.map((log) => ({
address: log.address,
data: log.data === '0x' ? null : fromHex(log.data.substring(2)),
block_height: log.blockNumber,
block_height: Number(log.blockNumber),
block_hash: log.blockHash,
tx_index: log.transactionIndex,
tx_index: Number(log.transactionIndex),
topic0: log.topics[0],
topic1: log.topics[1],
topic2: log.topics[2],
Expand All @@ -154,11 +152,11 @@ export default class CrawlEvmTransactionService extends BullableService {
nonce: offchainTx.nonce,
height: offchainTx.blockNumber,
index: offchainTx.transactionIndex,
gas_used: receiptTx.gasUsed,
gas_price: receiptTx.effectiveGasPrice,
gas_used: Number(receiptTx.gasUsed),
gas_price: Number(receiptTx.effectiveGasPrice),
gas: offchainTx.gas,
type: offchainTx.type,
status: receiptTx.status === 'success' ? 1 : 0,
status: Number(receiptTx.status),
contract_address: receiptTx.contractAddress,
value: offchainTx.value,
timestamp: offchainTx.timestamp,
Expand All @@ -184,14 +182,14 @@ export default class CrawlEvmTransactionService extends BullableService {
const promises = [];
for (let i = 0; i < blocks.length; i += 1) {
const block = blocks[i];
for (let j = 0; j < block.transactions.length; j += 1) {
const tx = block.transactions[j];
promises.push(
this.viemJsClient.getTransactionReceipt({ hash: tx.hash })
);
}
promises.push(
this.viemJsClient.request<any>({
method: 'eth_getBlockReceipts',
params: [`0x${block.height.toString(16)}`],
})
);
}
const receiptTxs = await Promise.all(promises);
const receiptTxs = _.flatten(await Promise.all(promises));
return receiptTxs as OpStackTransactionReceipt[];
}

Expand Down
92 changes: 52 additions & 40 deletions src/services/evm/evm_crawl_internal_tx.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import axios from 'axios';
import { Service } from '@ourparentcenter/moleculer-decorators-extended';
import { ServiceBroker } from 'moleculer';
import { PublicClient } from 'viem';
import { getViemClient } from '../../common/utils/etherjs_client';
import {
BlockCheckpoint,
EVMBlock,
Expand All @@ -10,19 +11,20 @@ import {
import { BULL_JOB_NAME, SERVICE } from './constant';
import BullableService, { QueueHandler } from '../../base/bullable.service';
import config from '../../../config.json' assert { type: 'json' };
import networks from '../../../network.json' assert { type: 'json' };
import knex from '../../common/utils/db_connection';

@Service({
name: SERVICE.V1.EVMCrawlInternalTx.key,
version: 1,
})
export default class EvmCrawlInternalTxService extends BullableService {
private selectedChain: any = networks.find(
(network) => network.chainId === config.chainId
);
viemJsClient!: PublicClient;

private EvmJsonRpc = this.selectedChain.EVMJSONRPC[0];
// private selectedChain: any = networks.find(
// (network) => network.chainId === config.chainId
// );

// private EvmJsonRpc = this.selectedChain.EVMJSONRPC[0];

public constructor(public broker: ServiceBroker) {
super(broker);
Expand Down Expand Up @@ -72,39 +74,45 @@ export default class EvmCrawlInternalTxService extends BullableService {
}

const requests = evmBlocks.map((evmBlock) =>
axios({
url: this.EvmJsonRpc,
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
data: {
jsonrpc: '2.0',
id: evmBlock.height,
method: 'debug_traceBlockByNumber',
params: [
`0x${evmBlock.height.toString(16)}`,
{
tracer: 'callTracer',
timeout: config.evmCrawlInternalTx.timeoutJSONRPC,
},
],
},
this.viemJsClient.request<any>({
method: 'debug_traceBlockByNumber',
params: [
`0x${evmBlock.height.toString(16)}`,
{
tracer: 'callTracer',
},
],
})
);

const responseBlocks = await Promise.all(requests).then((res) =>
res.map((result) => result.data)
);
// const requests = evmBlocks.map((evmBlock) =>
// axios({
// url: 'https://testnet.storyrpc.io',
// method: 'POST',
// headers: {
// Accept: 'application/json',
// 'Content-Type': 'application/json',
// },
// data: {
// jsonrpc: '2.0',
// id: evmBlock.height,
// method: 'debug_traceBlockByNumber',
// params: [
// `0x${evmBlock.height.toString(16)}`,
// {
// tracer: 'callTracer',
// timeout: config.evmCrawlInternalTx.timeoutJSONRPC,
// },
// ],
// },
// })
// );
const responseBlocks = await Promise.all(requests);
const internalTxSave: EvmInternalTransaction[] = [];

responseBlocks.forEach((responseBlock: any) => {
if (responseBlock.error) {
throw Error(JSON.stringify(responseBlock.error));
}
if (responseBlock.result) {
responseBlock.result.forEach((responseTx: any) => {
if (responseBlock) {
responseBlock.forEach((responseTx: any) => {
if (responseTx.result?.calls) {
const { txHash } = responseTx;
const evmTxDB = evmTxs.find((tx) => tx.hash === txHash);
Expand All @@ -124,13 +132,16 @@ export default class EvmCrawlInternalTxService extends BullableService {

await knex.transaction(async (trx) => {
blockCheckpoint.height = endBlock;
await trx
.batchInsert(
EvmInternalTransaction.tableName,
internalTxSave,
config.evmCrawlInternalTx.chunkSize
)
.transacting(trx);
if (internalTxSave.length > 0) {
await knex
.batchInsert(
EvmInternalTransaction.tableName,
internalTxSave,
config.evmCrawlInternalTx.chunkSize
)
.transacting(trx);
}

await BlockCheckpoint.query()
.transacting(trx)
.insert(blockCheckpoint)
Expand Down Expand Up @@ -178,6 +189,7 @@ export default class EvmCrawlInternalTxService extends BullableService {
}

public async _start(): Promise<void> {
this.viemJsClient = getViemClient();
await this.createJob(
BULL_JOB_NAME.EVM_CRAWL_INTERNAL_TX,
BULL_JOB_NAME.EVM_CRAWL_INTERNAL_TX,
Expand Down

0 comments on commit e007520

Please sign in to comment.