Feat: erc721 holder statistic + erc721 total supply

migrations/evm/20240912082032_erc721_holder_statistic.ts

@@ -0,0 +1,21 @@
+import { Knex } from 'knex';
+export async function up(knex: Knex): Promise<void> {
+ await knex.raw(`set statement_timeout to 0`);
+ await knex.raw(`
+ CREATE TABLE erc721_holder_statistic AS
+ select count(*), erc721_token.owner, erc721_token.erc721_contract_address
+ from erc721_token 
+ group by erc721_token.owner, erc721_token.erc721_contract_address;
+ ALTER TABLE erc721_holder_statistic ADD COLUMN id SERIAL PRIMARY KEY;
+ ALTER TABLE erc721_holder_statistic ADD COLUMN last_updated_height INTEGER;
+ CREATE INDEX erc721_holder_statistic_owner_index
+ ON erc721_holder_statistic (owner);
+ CREATE UNIQUE INDEX erc721_holder_statistic_erc721_contract_address_owner_index
+ ON erc721_holder_statistic (erc721_contract_address, owner);
+ CREATE INDEX erc721_holder_statistic_last_updated_height_index ON erc721_holder_statistic (last_updated_height)
+ `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+ await knex.schema.dropTableIfExists('erc721_holder_statistic');
+}

migrations/evm/20240916032201_erc721_contract_totalSupply.ts

@@ -0,0 +1,32 @@
+import { Knex } from 'knex';
+import { Erc721Token } from '../../src/models';
+import { ZERO_ADDRESS } from '../../src/services/evm/constant';
+
+export async function up(knex: Knex): Promise<void> {
+ await knex.schema.alterTable('erc721_contract', (table) => {
+ table.bigInteger('total_supply').defaultTo(0).index();
+ });
+ await knex.raw(`set statement_timeout to 0`);
+ const totalSupplies = await Erc721Token.query(knex)
+ .select('erc721_token.erc721_contract_address')
+ .where('erc721_token.owner', '!=', ZERO_ADDRESS)
+ .count()
+ .groupBy('erc721_token.erc721_contract_address');
+ if (totalSupplies.length > 0) {
+ const stringListUpdates = totalSupplies
+ .map(
+ (totalSuply) =>
+ `('${totalSuply.erc721_contract_address}', ${totalSuply.count})`
+ )
+ .join(',');
+ await knex.raw(
+ `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(address, total_supply) where temp.address = erc721_contract.address`
+ );
+ }
+}
+
+export async function down(knex: Knex): Promise<void> {
+ await knex.schema.alterTable('erc721_contract', (table) => {
+ table.dropColumn('total_supply');
+ });
+}

src/models/erc721_contract.ts

@@ -25,6 +25,8 @@ export class Erc721Contract extends BaseModel {
 
  last_updated_height!: number;
 
+ total_supply!: string;
+
  static get tableName() {
  return 'erc721_contract';
  }

src/models/erc721_holder_statistic.ts

@@ -0,0 +1,29 @@
+import BaseModel from './base';
+
+export class Erc721HolderStatistic extends BaseModel {
+ static softDelete = false;
+
+ erc721_contract_address!: string;
+
+ owner!: string;
+
+ count!: string;
+
+ last_updated_height!: number;
+
+ static get tableName() {
+ return 'erc721_holder_statistic';
+ }
+
+ static get jsonSchema() {
+ return {
+ type: 'object',
+ required: ['erc721_contract_address', 'owner', 'count'],
+ properties: {
+ erc721_contract_address: { type: 'string' },
+ owner: { type: 'string' },
+ count: { type: 'string' },
+ },
+ };
+ }
+}

src/models/index.ts

@@ -48,3 +48,4 @@ export * from './erc721_stats';
 export * from './evm_block';
 export * from './optimism_deposit';
 export * from './optimism_withdrawal';
+export * from './erc721_holder_statistic';

src/services/evm/erc721.service.ts

@@ -15,6 +15,7 @@ import {
  BlockCheckpoint,
  EVMSmartContract,
  Erc721Activity,
+ Erc721HolderStatistic,
  Erc721Stats,
  Erc721Token,
 } from '../../models';
@@ -103,7 +104,15 @@ export default class Erc721Service extends BullableService {
  if (erc721Activities.length > 0) {
  // create chunk array
  const listChunkErc721Activities = [];
- const erc721TokensOnDB: Erc721Token[] = [];
+ const erc721Contracts = _.keyBy(
+ await Erc721Contract.query()
+ .whereIn(
+ 'address',
+ erc721Activities.map((e) => e.erc721_contract_address)
+ )
+ .transacting(trx),
+ 'address'
+ );
  for (
  let i = 0;
  i < erc721Activities.length;
@@ -116,30 +125,57 @@ export default class Erc721Service extends BullableService {
  listChunkErc721Activities.push(chunk);
  }
  // process chunk array
- await Promise.all(
- // eslint-disable-next-line array-callback-return
- listChunkErc721Activities.map(async (chunk) => {
- const erc721TokensInChunk = await Erc721Token.query().whereIn(
- ['erc721_contract_address', 'token_id'],
- chunk.map((e) => [
- e.erc721_contract_address,
- // if token_id undefined (case approval_all), replace by null => not get any token (because token must have token_id)
- e.token_id || null,
- ])
- );
- erc721TokensOnDB.push(...erc721TokensInChunk);
- })
- );
-
+ const erc721TokensOnDB: Erc721Token[] = (
+ await Promise.all(
+ listChunkErc721Activities.map(async (chunk) =>
+ Erc721Token.query().whereIn(
+ ['erc721_contract_address', 'token_id'],
+ chunk.map((e) => [
+ e.erc721_contract_address,
+ // if token_id undefined (case approval_all), replace by null => not get any token (because token must have token_id)
+ e.token_id || null,
+ ])
+ )
+ )
+ )
+ ).flat();
+ // process chunk array
+ const erc721HolderStatsOnDB: Erc721HolderStatistic[] = (
+ await Promise.all(
+ listChunkErc721Activities.map(async (chunk) =>
+ Erc721HolderStatistic.query().whereIn(
+ ['erc721_contract_address', 'owner'],
+ _.uniqWith(
+ [
+ ...chunk.map((e) => [e.erc721_contract_address, e.from]),
+ ...chunk.map((e) => [e.erc721_contract_address, e.to]),
+ ],
+ _.isEqual
+ )
+ )
+ )
+ )
+ ).flat();
  const erc721Tokens = _.keyBy(
  erc721TokensOnDB,
  (o) => `${o.erc721_contract_address}_${o.token_id}`
  );
- const erc721Handler = new Erc721Handler(erc721Tokens, erc721Activities);
+ const erc721HolderStats = _.keyBy(
+ erc721HolderStatsOnDB,
+ (o) => `${o.erc721_contract_address}_${o.owner}`
+ );
+ const erc721Handler = new Erc721Handler(
+ erc721Contracts,
+ erc721Tokens,
+ erc721Activities,
+ erc721HolderStats
+ );
  erc721Handler.process();
  await Erc721Handler.updateErc721(
+ Object.values(erc721Handler.erc721Contracts),
  erc721Activities,
  Object.values(erc721Handler.erc721Tokens),
+ Object.values(erc721Handler.erc721HolderStats),
  trx
  );
  }
@@ -278,21 +314,6 @@ export default class Erc721Service extends BullableService {
  this.logger.info(`Reindex erc721 contract ${address} done.`);
  }
 
- @QueueHandler({
- queueName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
- jobName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
- })
- public async refreshErc721HolderStatistic(): Promise<void> {
- await knex.transaction(async (trx) => {
- await knex
- .raw(`set statement_timeout to ${config.erc721.statementTimeout}`)
- .transacting(trx);
- await knex.schema
- .refreshMaterializedView('m_view_erc721_holder_statistic')
- .transacting(trx);
- });
- }
-
  @Action({
  name: SERVICE.V1.Erc721.insertNewErc721Contracts.key,
  params: {
@@ -550,20 +571,6 @@ export default class Erc721Service extends BullableService {
  },
  }
  );
- await this.createJob(
- BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
- BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
- {},
- {
- removeOnComplete: true,
- removeOnFail: {
- count: 3,
- },
- repeat: {
- pattern: config.erc721.timeRefreshMViewErc721HolderStats,
- },
- }
- );
  }
  return super._start();
  }

src/services/evm/erc721_handler.ts

@@ -10,6 +10,7 @@ import {
  EVMTransaction,
  Erc721Activity,
  Erc721Contract,
+ Erc721HolderStatistic,
  Erc721Token,
  EvmEvent,
 } from '../../models';
@@ -64,18 +65,26 @@ export const ERC721_ACTION = {
  APPROVAL_FOR_ALL: 'approval_for_all',
 };
 export class Erc721Handler {
+ erc721Contracts: Dictionary<Erc721Contract>;
+
  // key: {contract_address}_{token_id}
  // value: erc721 token
  erc721Tokens: Dictionary<Erc721Token>;
 
  erc721Activities: Erc721Activity[];
 
+ erc721HolderStats: Dictionary<Erc721HolderStatistic>;
+
  constructor(
+ erc721Contracts: Dictionary<Erc721Contract>,
  erc721Tokens: Dictionary<Erc721Token>,
- erc721Activities: Erc721Activity[]
+ erc721Activities: Erc721Activity[],
+ erc721HolderStats: Dictionary<Erc721HolderStatistic>
  ) {
+ this.erc721Contracts = erc721Contracts;
  this.erc721Tokens = erc721Tokens;
  this.erc721Activities = erc721Activities;
+ this.erc721HolderStats = erc721HolderStats;
  }
 
  process() {
@@ -87,6 +96,8 @@ export class Erc721Handler {
  }
 
  handlerErc721Transfer(erc721Activity: Erc721Activity) {
+ const erc721Contract =
+ this.erc721Contracts[`${erc721Activity.erc721_contract_address}`];
  const token =
  this.erc721Tokens[
  `${erc721Activity.erc721_contract_address}_${erc721Activity.token_id}`
@@ -95,6 +106,11 @@ export class Erc721Handler {
  // update new owner and last updated height
  token.owner = erc721Activity.to;
  token.last_updated_height = erc721Activity.height;
+ if (erc721Activity.to === ZERO_ADDRESS) {
+ erc721Contract.total_supply = (
+ BigInt(erc721Contract.total_supply) - BigInt(1)
+ ).toString();
+ }
  } else if (erc721Activity.from === ZERO_ADDRESS) {
  // handle mint
  this.erc721Tokens[
@@ -106,9 +122,37 @@ export class Erc721Handler {
  last_updated_height: erc721Activity.height,
  burned: false,
  });
+ erc721Contract.total_supply = (
+ BigInt(erc721Contract.total_supply) + BigInt(1)
+ ).toString();
  } else {
  throw new Error('Handle erc721 tranfer error');
  }
+ // update erc721 holder statistics
+ const erc721HolderStatFrom =
+ this.erc721HolderStats[
+ `${erc721Activity.erc721_contract_address}_${erc721Activity.from}`
+ ];
+ this.erc721HolderStats[
+ `${erc721Activity.erc721_contract_address}_${erc721Activity.from}`
+ ] = Erc721HolderStatistic.fromJson({
+ erc721_contract_address: erc721Activity.erc721_contract_address,
+ owner: erc721Activity.from,
+ count: (BigInt(erc721HolderStatFrom?.count || 0) - BigInt(1)).toString(),
+ last_updated_height: erc721Activity.height,
+ });
+ const erc721HolderStatTo =
+ this.erc721HolderStats[
+ `${erc721Activity.erc721_contract_address}_${erc721Activity.to}`
+ ];
+ this.erc721HolderStats[
+ `${erc721Activity.erc721_contract_address}_${erc721Activity.to}`
+ ] = Erc721HolderStatistic.fromJson({
+ erc721_contract_address: erc721Activity.erc721_contract_address,
+ owner: erc721Activity.to,
+ count: (BigInt(erc721HolderStatTo?.count || 0) + BigInt(1)).toString(),
+ last_updated_height: erc721Activity.height,
+ });
  }
 
  static buildTransferActivity(
@@ -266,11 +310,28 @@ export class Erc721Handler {
  }
 
  static async updateErc721(
+ erc721Contracts: Erc721Contract[],
  erc721Activities: Erc721Activity[],
  erc721Tokens: Erc721Token[],
+ erc721HolderStats: Erc721HolderStatistic[],
  trx: Knex.Transaction
  ) {
+ // update erc721 contract: total supply
+ if (erc721Contracts.length > 0) {
+ const stringListUpdates = erc721Contracts
+ .map(
+ (erc721Contract) =>
+ `(${erc721Contract.id}, ${erc721Contract.total_supply})`
+ )
+ .join(',');
+ await knex
+ .raw(
+ `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(id, total_supply) where temp.id = erc721_contract.id`
+ )
+ .transacting(trx);
+ }
  let updatedTokens: Dictionary<Erc721Token> = {};
+ // update erc721 token: new token & new holder
  if (erc721Tokens.length > 0) {
  updatedTokens = _.keyBy(
  await Erc721Token.query()
@@ -290,6 +351,7 @@ export class Erc721Handler {
  (o) => `${o.erc721_contract_address}_${o.token_id}`
  );
  }
+ // insert new erc721 activities
  if (erc721Activities.length > 0) {
  erc721Activities.forEach((activity) => {
  const token =
@@ -309,6 +371,23 @@ export class Erc721Handler {
  )
  .transacting(trx);
  }
+ // update erc721 holder statistic
+ if (erc721HolderStats.length > 0) {
+ await Erc721HolderStatistic.query()
+ .transacting(trx)
+ .insert(
+ erc721HolderStats.map((e) =>
+ Erc721HolderStatistic.fromJson({
+ erc721_contract_address: e.erc721_contract_address,
+ owner: e.owner,
+ count: e.count,
+ last_updated_height: e.last_updated_height,
+ })
+ )
+ )
+ .onConflict(['erc721_contract_address', 'owner'])
+ .merge();
+ }
  }
 
  static async calErc721Stats(addresses?: string[]): Promise<Erc721Contract[]> {