diff --git a/.docker/docker-entrypoint-initdb.d/init.txt b/.docker/docker-entrypoint-initdb.d/init.txt index 889d71d1..f0263b17 100644 --- a/.docker/docker-entrypoint-initdb.d/init.txt +++ b/.docker/docker-entrypoint-initdb.d/init.txt @@ -44,7 +44,7 @@ CREATE TYPE block_result AS ( "size" int4, "gasLimit" int8, "gasUsed" int8, - "mixHash" hash, + "mixHash" hash, "timestamp" int4 ); DROP TYPE IF EXISTS filter_result CASCADE; @@ -221,7 +221,7 @@ BEGIN hash, -- hash parent_hash, -- parentHash repeat('\000', 8)::bytea, -- nonce - repeat('\000', 32)::bytea, -- sha3Uncles + '\x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347', -- sha3Uncles keccak256(rlp.encode([])) repeat('\000', 256)::bytea, -- logsBloom transactions_root, -- transactionsRoot state_root, -- stateRoot diff --git a/CHANGES.md b/CHANGES.md index 08a89ca2..1c9eeb27 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## 2021-12-14 + +- Run `node lib/data_migrations/2021-12-14-empty-blocks.js` to reindex all historical empty blocks. +In addition, reload the following stored procedures: + +- `etc/schema/functions/eth_getBlockByNumber.sql` + +or + +```bash +$ cd migrations +$ goose postgres "user= password= dbname=aurora sslmode=disable" up +``` + ## 2021-12-08 A new column `from` was added into the `event` table. Check out this PR [#120](https://github.com/aurora-is-near/aurora-relayer/pull/120) for more info. diff --git a/etc/schema/functions/eth_getBlockByNumber.sql b/etc/schema/functions/eth_getBlockByNumber.sql index e0242b6a..d2af2dfc 100644 --- a/etc/schema/functions/eth_getBlockByNumber.sql +++ b/etc/schema/functions/eth_getBlockByNumber.sql @@ -9,7 +9,7 @@ BEGIN hash, -- hash parent_hash, -- parentHash repeat('\000', 8)::bytea, -- nonce - repeat('\000', 32)::bytea, -- sha3Uncles + '\x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347', -- sha3Uncles keccak256(rlp.encode([])) repeat('\000', 256)::bytea, -- logsBloom transactions_root, -- transactionsRoot state_root, -- stateRoot diff --git a/lib/data_migrations/2021-12-14-empty-blocks.d.ts b/lib/data_migrations/2021-12-14-empty-blocks.d.ts new file mode 100644 index 00000000..ddff3dcd --- /dev/null +++ b/lib/data_migrations/2021-12-14-empty-blocks.d.ts @@ -0,0 +1,7 @@ +import { ConnectEnv } from '@aurora-is-near/engine'; +declare global { + namespace NodeJS { + interface ProcessEnv extends ConnectEnv { + } + } +} diff --git a/lib/data_migrations/2021-12-14-empty-blocks.js b/lib/data_migrations/2021-12-14-empty-blocks.js new file mode 100644 index 00000000..c1875db1 --- /dev/null +++ b/lib/data_migrations/2021-12-14-empty-blocks.js @@ -0,0 +1,77 @@ +/* This is free and unencumbered software released into the public domain. */ +import { parseConfig } from '../config.js'; +import { pg, sql } from '../database.js'; +import { emptyTransactionsRoot } from '../utils.js'; +import { AccountID, Engine, } from '@aurora-is-near/engine'; +import { program } from 'commander'; +import externalConfig from 'config'; +import pino from 'pino'; +const logger = pino(pino.destination(2)); +class ReindexWorker { + constructor(config, network, logger, engine) { + this.config = config; + this.network = network; + this.logger = logger; + this.engine = engine; + this.contractID = AccountID.parse(this.config.engine).unwrap(); + this.pgClient = new pg.Client(config.database); + } + async run(blockId) { + await this.pgClient.connect(); + const step = Number(this.config.batchSize || 1000); + let startBlockQuery = sql + .select('block.id') + .from('block') + .order('block.id DESC') + .limit(1); + if (blockId > 0) { + startBlockQuery = startBlockQuery.where(sql(`block.id <= $1`, blockId)); + } + const transactionQueryResult = await this.pgClient.query(startBlockQuery.toParams()); + const startBlockId = transactionQueryResult.rows[0].id; + for (let blockId = startBlockId; blockId > 0; blockId -= step) { + const endBlockId = blockId - step > 0 ? blockId - step : 0; + logger.info(`Fetching blocks ${endBlockId}..${blockId}`); + const updateQuery = sql + .update('block', { transactions_root: emptyTransactionsRoot() }) + .where(sql('(SELECT COUNT(id) FROM transaction t WHERE t.block = block.id) = 0')) + .where(sql(`block.id >= $1 AND block.id < $2`, blockId - step, blockId)); + await this.pgClient.query(updateQuery.toParams()); + } + process.exit(0); // EX_OK + } +} +async function main(argv, env) { + program + .option('-d, --debug', 'enable debug output') + .option('-v, --verbose', 'enable verbose output') + .option('--network ', `specify NEAR network ID (default: "${env.NEAR_ENV || 'local'}")`) + .option('--engine ', `specify Aurora Engine account ID (default: "${env.AURORA_ENGINE || 'aurora.test.near'}")`) + .option('-B, --block ', `specify block height to begin indexing from (default: 0)`) + .option('--batch-size ', `specify batch size for fetching block metadata (default: 1000)`) + .parse(argv); + const opts = program.opts(); + const [network, config] = parseConfig(opts, externalConfig, env); + const blockID = opts.block !== undefined ? parseInt(opts.block) : 0; + if (config.debug) { + for (const source of externalConfig.util.getConfigSources()) { + console.error(`Loaded configuration file ${source.name}.`); + } + console.error('Configuration:', config); + } + logger.info('starting reindexing of transactions'); + const engine = await Engine.connect({ + network: network.id, + endpoint: config.endpoint, + contract: config.engine, + }, env); + const indexer = new ReindexWorker(config, network, logger, engine); + await indexer.run(blockID); +} +main(process.argv, process.env).catch((error) => { + const errorMessage = error.message.startsWith('<') + ? error.name + : error.message; + logger.error(errorMessage); + process.exit(70); // EX_SOFTWARE +}); diff --git a/lib/indexer_worker.js b/lib/indexer_worker.js index 5aec4237..a0492cb2 100644 --- a/lib/indexer_worker.js +++ b/lib/indexer_worker.js @@ -2,7 +2,7 @@ import { parentPort, workerData } from 'worker_threads'; import { pg, sql } from './database.js'; import format from 'pg-format'; -import { computeBlockHash, generateEmptyBlock } from './utils.js'; +import { computeBlockHash, generateEmptyBlock, emptyTransactionsRoot, } from './utils.js'; import { AccountID, Engine, hexToBytes, } from '@aurora-is-near/engine'; export class Indexer { constructor(config, network, engine) { @@ -61,6 +61,9 @@ export class Indexer { const block = block_.getMetadata(); const blockHash = computeBlockHash(block.number, this.contractID.toString(), this.network.chainID); const parentHash = computeBlockHash(block.number - 1, this.contractID.toString(), this.network.chainID); + const transactionsRoot = block.transactions.length == 0 + ? emptyTransactionsRoot() + : block.transactionsRoot; const query = sql.insert('block', { chain: this.network.chainID, id: block.number, @@ -71,7 +74,7 @@ export class Indexer { gas_limit: 0, gas_used: 0, parent_hash: parentHash, - transactions_root: block.transactionsRoot, + transactions_root: transactionsRoot, state_root: block.stateRoot, receipts_root: block.receiptsRoot, }); @@ -164,7 +167,7 @@ export class Indexer { blockId: blockID, index: transactionIndex, }); - this.pgClient.query(`NOTIFY log, ${format.literal(logDetails)}`); + await this.pgClient.query(`NOTIFY log, ${format.literal(logDetails)}`); } catch (error) { console.error('indexEvent', error); @@ -176,7 +179,7 @@ export class Indexer { } for (;;) { if (await this.isBlockIndexed(this.pendingHeadBlock)) { - this.pgClient.query(`NOTIFY block, ${format.literal(this.pendingHeadBlock.toString())}`); + await this.pgClient.query(`NOTIFY block, ${format.literal(this.pendingHeadBlock.toString())}`); this.pendingHeadBlock += 1; } else { diff --git a/lib/utils.d.ts b/lib/utils.d.ts index be77098d..df24ac71 100644 --- a/lib/utils.d.ts +++ b/lib/utils.d.ts @@ -15,3 +15,4 @@ export declare type EmptyBlock = { }; export declare function computeBlockHash(blockHeight: number, accountId: string, chainId: number): Buffer; export declare function generateEmptyBlock(blockHeight: number, accountId: string, chainId: number): EmptyBlock; +export declare function emptyTransactionsRoot(): Buffer; diff --git a/lib/utils.js b/lib/utils.js index 580cf2d1..6b2c3cbd 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,5 +1,7 @@ /* This is free and unencumbered software released into the public domain. */ import { sha256 } from 'ethereum-cryptography/sha256.js'; +import { keccak256 } from 'ethereumjs-util'; +import * as rlp from 'rlp'; export function computeBlockHash(blockHeight, accountId, chainId) { return sha256(generateBlockPreImage(blockHeight, accountId, chainId)); } @@ -29,8 +31,11 @@ export function generateEmptyBlock(blockHeight, accountId, chainId) { gasLimit: 0, gasUsed: 0, parentHash: parentHash, - transactionsRoot: Buffer.alloc(32), + transactionsRoot: emptyTransactionsRoot(), stateRoot: Buffer.alloc(32), receiptsRoot: Buffer.alloc(32), }; } +export function emptyTransactionsRoot() { + return keccak256(rlp.encode('')); +} diff --git a/migrations/20211215003309_change_sha3_uncles_in_eth_getblockbynumber.sql b/migrations/20211215003309_change_sha3_uncles_in_eth_getblockbynumber.sql new file mode 100644 index 00000000..8203cbaa --- /dev/null +++ b/migrations/20211215003309_change_sha3_uncles_in_eth_getblockbynumber.sql @@ -0,0 +1,67 @@ +-- +goose Up +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION eth_getBlockByNumber(block_id blockno) RETURNS block_result AS $$ +DECLARE + result block_result; +BEGIN + SELECT + id, -- number + hash, -- hash + parent_hash, -- parentHash + repeat('\000', 8)::bytea, -- nonce + '\x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347', -- sha3Uncles keccak256(rlp.encode([])) + repeat('\000', 256)::bytea, -- logsBloom + transactions_root, -- transactionsRoot + state_root, -- stateRoot + receipts_root, -- receiptsRoot + repeat('\000', 20)::bytea, -- miner + 0, -- difficulty + 0, -- totalDifficulty + ''::bytea, -- extraData + size, -- size + gas_limit, -- gasLimit + gas_used, -- gasUsed + repeat('\000', 32)::hash, -- mixHash + COALESCE(EXTRACT(EPOCH FROM timestamp), 0)::int4 -- timestamp + FROM block + WHERE id = block_id + LIMIT 1 + INTO STRICT result; + RETURN result; +END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION eth_getBlockByNumber(block_id blockno) RETURNS block_result AS $$ +DECLARE + result block_result; +BEGIN + SELECT + id, -- number + hash, -- hash + parent_hash, -- parentHash + repeat('\000', 8)::bytea, -- nonce + repeat('\000', 32)::bytea, -- sha3Uncles + repeat('\000', 256)::bytea, -- logsBloom + transactions_root, -- transactionsRoot + state_root, -- stateRoot + receipts_root, -- receiptsRoot + repeat('\000', 20)::bytea, -- miner + 0, -- difficulty + 0, -- totalDifficulty + ''::bytea, -- extraData + size, -- size + gas_limit, -- gasLimit + gas_used, -- gasUsed + repeat('\000', 32)::hash, -- mixHash + COALESCE(EXTRACT(EPOCH FROM timestamp), 0)::int4 -- timestamp + FROM block + WHERE id = block_id + LIMIT 1 + INTO STRICT result; + RETURN result; +END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; +-- +goose StatementEnd diff --git a/package-lock.json b/package-lock.json index dcabfa00..1afe7f31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "node-worker-threads-pool": "^1.5.1", "pg": "^8.7.1", "pg-format": "^1.0.4", - "rlp": "^2.2.6", + "rlp": "^2.2.7", "sql-bricks-postgres": "^0.5.0", "ts-jest": "^27.0.3", "ws": "^8.2.3" @@ -7960,21 +7960,16 @@ } }, "node_modules/rlp": { - "version": "2.2.6", - "resolved": "https://registry.npmjs.org/rlp/-/rlp-2.2.6.tgz", - "integrity": "sha512-HAfAmL6SDYNWPUOJNrM500x4Thn4PZsEy5pijPh40U9WfNk0z15hUYzO9xVIMAdIHdFtD8CBDHd75Td1g36Mjg==", + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/rlp/-/rlp-2.2.7.tgz", + "integrity": "sha512-d5gdPmgQ0Z+AklL2NVXr/IoSjNZFfTVvQWzL/AM2AOcSzYP2xjlb0AC8YyCLc41MSNf6P6QVtjgPdmVtzb+4lQ==", "dependencies": { - "bn.js": "^4.11.1" + "bn.js": "^5.2.0" }, "bin": { "rlp": "bin/rlp" } }, - "node_modules/rlp/node_modules/bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -15397,18 +15392,11 @@ } }, "rlp": { - "version": "2.2.6", - "resolved": "https://registry.npmjs.org/rlp/-/rlp-2.2.6.tgz", - "integrity": "sha512-HAfAmL6SDYNWPUOJNrM500x4Thn4PZsEy5pijPh40U9WfNk0z15hUYzO9xVIMAdIHdFtD8CBDHd75Td1g36Mjg==", + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/rlp/-/rlp-2.2.7.tgz", + "integrity": "sha512-d5gdPmgQ0Z+AklL2NVXr/IoSjNZFfTVvQWzL/AM2AOcSzYP2xjlb0AC8YyCLc41MSNf6P6QVtjgPdmVtzb+4lQ==", "requires": { - "bn.js": "^4.11.1" - }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } + "bn.js": "^5.2.0" } }, "run-parallel": { diff --git a/package.json b/package.json index 05fc7b42..95648cdc 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "node-worker-threads-pool": "^1.5.1", "pg": "^8.7.1", "pg-format": "^1.0.4", - "rlp": "^2.2.6", + "rlp": "^2.2.7", "sql-bricks-postgres": "^0.5.0", "ts-jest": "^27.0.3", "ws": "^8.2.3" diff --git a/src/data_migrations/2021-12-14-empty-blocks.ts b/src/data_migrations/2021-12-14-empty-blocks.ts new file mode 100644 index 00000000..0515323e --- /dev/null +++ b/src/data_migrations/2021-12-14-empty-blocks.ts @@ -0,0 +1,135 @@ +/* This is free and unencumbered software released into the public domain. */ + +import { Config, parseConfig } from '../config.js'; +import { pg, sql } from '../database.js'; +import { emptyTransactionsRoot } from '../utils.js'; + +import { + AccountID, + Engine, + ConnectEnv, + NetworkConfig, +} from '@aurora-is-near/engine'; +import { program } from 'commander'; +import externalConfig from 'config'; +import pino, { Logger } from 'pino'; + +const logger = pino(pino.destination(2)); + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace NodeJS { + // eslint-disable-next-line @typescript-eslint/no-empty-interface + interface ProcessEnv extends ConnectEnv {} + } +} + +class ReindexWorker { + protected readonly contractID: AccountID; + protected readonly pgClient: pg.Client; + + constructor( + public readonly config: Config, + public readonly network: NetworkConfig, + public readonly logger: Logger, + public readonly engine: Engine + ) { + this.contractID = AccountID.parse(this.config.engine).unwrap(); + this.pgClient = new pg.Client(config.database); + } + + async run(blockId: number): Promise { + await this.pgClient.connect(); + const step = Number(this.config.batchSize || 1000); + + let startBlockQuery = sql + .select('block.id') + .from('block') + .order('block.id DESC') + .limit(1); + if (blockId > 0) { + startBlockQuery = startBlockQuery.where(sql(`block.id <= $1`, blockId)); + } + const transactionQueryResult = await this.pgClient.query( + startBlockQuery.toParams() + ); + const startBlockId = transactionQueryResult.rows[0].id; + + for (let blockId = startBlockId; blockId > 0; blockId -= step) { + const endBlockId = blockId - step > 0 ? blockId - step : 0; + logger.info(`Fetching blocks ${endBlockId}..${blockId}`); + const updateQuery = sql + .update('block', { transactions_root: emptyTransactionsRoot() }) + .where( + sql( + '(SELECT COUNT(id) FROM transaction t WHERE t.block = block.id) = 0' + ) + ) + .where( + sql(`block.id >= $1 AND block.id < $2`, blockId - step, blockId) + ); + await this.pgClient.query(updateQuery.toParams()); + } + process.exit(0); // EX_OK + } +} + +async function main(argv: string[], env: NodeJS.ProcessEnv) { + program + .option('-d, --debug', 'enable debug output') + .option('-v, --verbose', 'enable verbose output') + .option( + '--network ', + `specify NEAR network ID (default: "${env.NEAR_ENV || 'local'}")` + ) + .option( + '--engine ', + `specify Aurora Engine account ID (default: "${ + env.AURORA_ENGINE || 'aurora.test.near' + }")` + ) + .option( + '-B, --block ', + `specify block height to begin indexing from (default: 0)` + ) + .option( + '--batch-size ', + `specify batch size for fetching block metadata (default: 1000)` + ) + .parse(argv); + + const opts = program.opts() as Config; + const [network, config] = parseConfig( + opts, + externalConfig as unknown as Config, + env + ); + const blockID = opts.block !== undefined ? parseInt(opts.block as string) : 0; + + if (config.debug) { + for (const source of externalConfig.util.getConfigSources()) { + console.error(`Loaded configuration file ${source.name}.`); + } + console.error('Configuration:', config); + } + + logger.info('starting reindexing of transactions'); + const engine = await Engine.connect( + { + network: network.id, + endpoint: config.endpoint, + contract: config.engine, + }, + env + ); + const indexer = new ReindexWorker(config, network, logger, engine); + await indexer.run(blockID); +} + +main(process.argv, process.env).catch((error: Error) => { + const errorMessage = error.message.startsWith('<') + ? error.name + : error.message; + logger.error(errorMessage); + process.exit(70); // EX_SOFTWARE +}); diff --git a/src/indexer_worker.ts b/src/indexer_worker.ts index b034eebd..cfa8db0f 100644 --- a/src/indexer_worker.ts +++ b/src/indexer_worker.ts @@ -5,7 +5,12 @@ import { MessagePort, parentPort, workerData } from 'worker_threads'; import { Config } from './config.js'; import { pg, sql } from './database.js'; import format from 'pg-format'; -import { computeBlockHash, EmptyBlock, generateEmptyBlock } from './utils.js'; +import { + computeBlockHash, + EmptyBlock, + generateEmptyBlock, + emptyTransactionsRoot, +} from './utils.js'; import { AccountID, BlockHeight, @@ -98,6 +103,11 @@ export class Indexer { this.contractID.toString(), this.network.chainID ); + + const transactionsRoot = + block.transactions.length == 0 + ? emptyTransactionsRoot() + : block.transactionsRoot; const query = sql.insert('block', { chain: this.network.chainID, id: block.number, @@ -108,7 +118,7 @@ export class Indexer { gas_limit: 0, // FIXME: block.gasLimit, gas_used: 0, // FIXME: block.gasUsed, parent_hash: parentHash, - transactions_root: block.transactionsRoot, + transactions_root: transactionsRoot, state_root: block.stateRoot, receipts_root: block.receiptsRoot, }); @@ -235,7 +245,7 @@ export class Indexer { blockId: blockID, index: transactionIndex, }); - this.pgClient.query(`NOTIFY log, ${format.literal(logDetails)}`); + await this.pgClient.query(`NOTIFY log, ${format.literal(logDetails)}`); } catch (error) { console.error('indexEvent', error); } @@ -247,7 +257,7 @@ export class Indexer { } for (;;) { if (await this.isBlockIndexed(this.pendingHeadBlock)) { - this.pgClient.query( + await this.pgClient.query( `NOTIFY block, ${format.literal(this.pendingHeadBlock.toString())}` ); this.pendingHeadBlock += 1; diff --git a/src/utils.ts b/src/utils.ts index 9280ae11..a06d0975 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,8 @@ /* This is free and unencumbered software released into the public domain. */ import { sha256 } from 'ethereum-cryptography/sha256.js'; +import { keccak256 } from 'ethereumjs-util'; +import * as rlp from 'rlp'; export type EmptyBlock = { chain: number; @@ -62,8 +64,12 @@ export function generateEmptyBlock( gasLimit: 0, gasUsed: 0, parentHash: parentHash, - transactionsRoot: Buffer.alloc(32), + transactionsRoot: emptyTransactionsRoot(), stateRoot: Buffer.alloc(32), receiptsRoot: Buffer.alloc(32), }; } + +export function emptyTransactionsRoot(): Buffer { + return keccak256(rlp.encode('')); +} diff --git a/tsconfig.json b/tsconfig.json index aeed2e36..d004cee9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "src/prehistory.ts", "src/decs.d.ts", "src/data_migrations/2021-12-02-event.ts", + "src/data_migrations/2021-12-14-empty-blocks.ts", ], "extends": "@tsconfig/node14/tsconfig.json", "compilerOptions": {