Feat: erc721 holder statistic + erc721 total supply

migrations/evm/20240912082032_erc721_holder_statistic.ts

@@ -0,0 +1,19 @@
+import { Knex } from 'knex';
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`set statement_timeout to 0`);
+  await knex.raw(`
+    CREATE TABLE erc721_holder_statistic AS
+      select count(*), erc721_token.owner, erc721_token.erc721_contract_address
+      from erc721_token 
+      group by erc721_token.owner, erc721_token.erc721_contract_address;
+    ALTER TABLE erc721_holder_statistic ADD COLUMN id SERIAL PRIMARY KEY;
+    CREATE INDEX erc721_holder_statistic_owner_index
+      ON erc721_holder_statistic (owner);
+    CREATE UNIQUE INDEX erc721_holder_statistic_erc721_contract_address_owner_index
+      ON erc721_holder_statistic (erc721_contract_address, owner);
+  `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists('erc721_holder_statistic');
+}

migrations/evm/20240916032201_erc721_contract_totalSupply.ts

@@ -0,0 +1,32 @@
+import { Knex } from 'knex';
+import { Erc721Token } from '../../src/models';
+import { ZERO_ADDRESS } from '../../src/services/evm/constant';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc721_contract', (table) => {
+    table.bigInteger('total_supply').defaultTo(0).index();
+  });
+  await knex.raw(`set statement_timeout to 0`);
+  const totalSupplies = await Erc721Token.query(knex)
+    .select('erc721_token.erc721_contract_address')
+    .where('erc721_token.owner', '!=', ZERO_ADDRESS)
+    .count()
+    .groupBy('erc721_token.erc721_contract_address');
+  if (totalSupplies.length > 0) {
+    const stringListUpdates = totalSupplies
+      .map(
+        (totalSuply) =>
+          `('${totalSuply.erc721_contract_address}', ${totalSuply.count})`
+      )
+      .join(',');
+    await knex.raw(
+      `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(address, total_supply) where temp.address = erc721_contract.address`
+    );
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('erc721_contract', (table) => {
+    table.dropColumn('total_supply');
+  });
+}

src/models/erc721_contract.ts

@@ -25,6 +25,8 @@ export class Erc721Contract extends BaseModel {
 
   last_updated_height!: number;
 
+  total_supply!: string;
+
   static get tableName() {
     return 'erc721_contract';
   }

src/models/erc721_holder_statistic.ts

@@ -0,0 +1,27 @@
+import BaseModel from './base';
+
+export class Erc721HolderStatistic extends BaseModel {
+  static softDelete = false;
+
+  erc721_contract_address!: string;
+
+  owner!: string;
+
+  count!: string;
+
+  static get tableName() {
+    return 'erc721_holder_statistic';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: ['erc721_contract_address', 'owner', 'count'],
+      properties: {
+        erc721_contract_address: { type: 'string' },
+        owner: { type: 'string' },
+        count: { type: 'string' },
+      },
+    };
+  }
+}

src/models/index.ts

@@ -48,3 +48,4 @@ export * from './erc721_stats';
 export * from './evm_block';
 export * from './optimism_deposit';
 export * from './optimism_withdrawal';
+export * from './erc721_holder_statistic';

src/services/evm/erc721.service.ts

@@ -15,6 +15,7 @@ import {
   BlockCheckpoint,
   EVMSmartContract,
   Erc721Activity,
+  Erc721HolderStatistic,
   Erc721Stats,
   Erc721Token,
 } from '../../models';
@@ -103,6 +104,16 @@ export default class Erc721Service extends BullableService {
       if (erc721Activities.length > 0) {
         // create chunk array
         const listChunkErc721Activities = [];
+        const erc721Contracts = _.keyBy(
+          await Erc721Contract.query()
+            .whereIn(
+              'address',
+              erc721Activities.map((e) => e.erc721_contract_address)
+            )
+            .transacting(trx),
+          'address'
+        );
+        const erc721HolderStatsOnDB: Erc721HolderStatistic[] = [];
         const erc721TokensOnDB: Erc721Token[] = [];
         for (
           let i = 0;
@@ -130,16 +141,44 @@ export default class Erc721Service extends BullableService {
             erc721TokensOnDB.push(...erc721TokensInChunk);
           })
         );
-
+        // process chunk array
+        await Promise.all(
+          // eslint-disable-next-line array-callback-return
+          listChunkErc721Activities.map(async (chunk) => {
+            const erc721HolderStatsInChunk =
+              await Erc721HolderStatistic.query().whereIn(
+                ['erc721_contract_address', 'owner'],
+                _.uniqWith(
+                  [
+                    ...chunk.map((e) => [e.erc721_contract_address, e.from]),
+                    ...chunk.map((e) => [e.erc721_contract_address, e.to]),
+                  ],
+                  _.isEqual
+                )
+              );
+            erc721HolderStatsOnDB.push(...erc721HolderStatsInChunk);
+          })
+        );
         const erc721Tokens = _.keyBy(
           erc721TokensOnDB,
           (o) => `${o.erc721_contract_address}_${o.token_id}`
         );
-        const erc721Handler = new Erc721Handler(erc721Tokens, erc721Activities);
+        const erc721HolderStats = _.keyBy(
+          erc721HolderStatsOnDB,
+          (o) => `${o.erc721_contract_address}_${o.owner}`
+        );
+        const erc721Handler = new Erc721Handler(
+          erc721Contracts,
+          erc721Tokens,
+          erc721Activities,
+          erc721HolderStats
+        );
         erc721Handler.process();
         await Erc721Handler.updateErc721(
+          Object.values(erc721Handler.erc721Contracts),
           erc721Activities,
           Object.values(erc721Handler.erc721Tokens),
+          Object.values(erc721Handler.erc721HolderStats),
           trx
         );
       }
@@ -278,21 +317,6 @@ export default class Erc721Service extends BullableService {
     this.logger.info(`Reindex erc721 contract ${address} done.`);
   }
 
-  @QueueHandler({
-    queueName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
-    jobName: BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
-  })
-  public async refreshErc721HolderStatistic(): Promise<void> {
-    await knex.transaction(async (trx) => {
-      await knex
-        .raw(`set statement_timeout to ${config.erc721.statementTimeout}`)
-        .transacting(trx);
-      await knex.schema
-        .refreshMaterializedView('m_view_erc721_holder_statistic')
-        .transacting(trx);
-    });
-  }
-
   @Action({
     name: SERVICE.V1.Erc721.insertNewErc721Contracts.key,
     params: {
@@ -550,20 +574,6 @@ export default class Erc721Service extends BullableService {
           },
         }
       );
-      await this.createJob(
-        BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
-        BULL_JOB_NAME.REFRESH_ERC721_HOLDER_STATISTIC,
-        {},
-        {
-          removeOnComplete: true,
-          removeOnFail: {
-            count: 3,
-          },
-          repeat: {
-            pattern: config.erc721.timeRefreshMViewErc721HolderStats,
-          },
-        }
-      );
     }
     return super._start();
   }

src/services/evm/erc721_handler.ts

@@ -10,6 +10,7 @@ import {
   EVMTransaction,
   Erc721Activity,
   Erc721Contract,
+  Erc721HolderStatistic,
   Erc721Token,
   EvmEvent,
 } from '../../models';
@@ -64,18 +65,26 @@ export const ERC721_ACTION = {
   APPROVAL_FOR_ALL: 'approval_for_all',
 };
 export class Erc721Handler {
+  erc721Contracts: Dictionary<Erc721Contract>;
+
   // key: {contract_address}_{token_id}
   // value: erc721 token
   erc721Tokens: Dictionary<Erc721Token>;
 
   erc721Activities: Erc721Activity[];
 
+  erc721HolderStats: Dictionary<Erc721HolderStatistic>;
+
   constructor(
+    erc721Contracts: Dictionary<Erc721Contract>,
     erc721Tokens: Dictionary<Erc721Token>,
-    erc721Activities: Erc721Activity[]
+    erc721Activities: Erc721Activity[],
+    erc721HolderStats: Dictionary<Erc721HolderStatistic>
   ) {
+    this.erc721Contracts = erc721Contracts;
     this.erc721Tokens = erc721Tokens;
     this.erc721Activities = erc721Activities;
+    this.erc721HolderStats = erc721HolderStats;
   }
 
   process() {
@@ -87,6 +96,8 @@ export class Erc721Handler {
   }
 
   handlerErc721Transfer(erc721Activity: Erc721Activity) {
+    const erc721Contract =
+      this.erc721Contracts[`${erc721Activity.erc721_contract_address}`];
     const token =
       this.erc721Tokens[
         `${erc721Activity.erc721_contract_address}_${erc721Activity.token_id}`
@@ -95,6 +106,11 @@ export class Erc721Handler {
       // update new owner and last updated height
       token.owner = erc721Activity.to;
       token.last_updated_height = erc721Activity.height;
+      if (erc721Activity.to === ZERO_ADDRESS) {
+        erc721Contract.total_supply = (
+          BigInt(erc721Contract.total_supply) - BigInt(1)
+        ).toString();
+      }
     } else if (erc721Activity.from === ZERO_ADDRESS) {
       // handle mint
       this.erc721Tokens[
@@ -106,9 +122,42 @@ export class Erc721Handler {
         last_updated_height: erc721Activity.height,
         burned: false,
       });
+      erc721Contract.total_supply = (
+        BigInt(erc721Contract.total_supply) + BigInt(1)
+      ).toString();
     } else {
       throw new Error('Handle erc721 tranfer error');
     }
+    // update erc721 holder statistics
+    if (erc721Activity.from !== ZERO_ADDRESS) {
+      const erc721HolderStatFrom =
+        this.erc721HolderStats[
+          `${erc721Activity.erc721_contract_address}_${erc721Activity.from}`
+        ];
+      if (!erc721HolderStatFrom) {
+        throw new Error(
+          `Erc721 holder ${erc721Activity.from} havent been audited`
+        );
+      }
+      erc721HolderStatFrom.count = (
+        BigInt(erc721HolderStatFrom.count) - BigInt(1)
+      ).toString();
+    }
+    if (erc721Activity.to !== ZERO_ADDRESS) {
+      const erc721HolderStatTo =
+        this.erc721HolderStats[
+          `${erc721Activity.erc721_contract_address}_${erc721Activity.to}`
+        ];
+      this.erc721HolderStats[
+        `${erc721Activity.erc721_contract_address}_${erc721Activity.to}`
+      ] = Erc721HolderStatistic.fromJson({
+        erc721_contract_address: erc721Activity.erc721_contract_address,
+        owner: erc721Activity.to,
+        count: erc721HolderStatTo
+          ? (BigInt(erc721HolderStatTo.count) + BigInt(1)).toString()
+          : BigInt(1).toString(),
+      });
+    }
   }
 
   static buildTransferActivity(
@@ -266,11 +315,28 @@ export class Erc721Handler {
   }
 
   static async updateErc721(
+    erc721Contracts: Erc721Contract[],
     erc721Activities: Erc721Activity[],
     erc721Tokens: Erc721Token[],
+    erc721HolderStats: Erc721HolderStatistic[],
     trx: Knex.Transaction
   ) {
+    // update erc721 contract: total supply
+    if (erc721Contracts.length > 0) {
+      const stringListUpdates = erc721Contracts
+        .map(
+          (erc721Contract) =>
+            `(${erc721Contract.id}, ${erc721Contract.total_supply})`
+        )
+        .join(',');
+      await knex
+        .raw(
+          `UPDATE erc721_contract SET total_supply = temp.total_supply from (VALUES ${stringListUpdates}) as temp(id, total_supply) where temp.id = erc721_contract.id`
+        )
+        .transacting(trx);
+    }
     let updatedTokens: Dictionary<Erc721Token> = {};
+    // update erc721 token: new token & new holder
     if (erc721Tokens.length > 0) {
       updatedTokens = _.keyBy(
         await Erc721Token.query()
@@ -290,6 +356,7 @@ export class Erc721Handler {
         (o) => `${o.erc721_contract_address}_${o.token_id}`
       );
     }
+    // insert new erc721 activities
     if (erc721Activities.length > 0) {
       erc721Activities.forEach((activity) => {
         const token =
@@ -309,6 +376,22 @@ export class Erc721Handler {
         )
         .transacting(trx);
     }
+    // update erc721 holder statistic
+    if (erc721HolderStats.length > 0) {
+      await Erc721HolderStatistic.query()
+        .transacting(trx)
+        .insert(
+          erc721HolderStats.map((e) =>
+            Erc721HolderStatistic.fromJson({
+              erc721_contract_address: e.erc721_contract_address,
+              owner: e.owner,
+              count: e.count,
+            })
+          )
+        )
+        .onConflict(['erc721_contract_address', 'owner'])
+        .merge();
+    }
   }
 
   static async calErc721Stats(addresses?: string[]): Promise<Erc721Contract[]> {