Feat/statistics (#272)

* feat: move statistic feature from v1 to v2 (not tested yet)

* feat: change job params so it can aggregate data at a specific date

* fix: query fields from exact table

* fix: query exact fields

* feat: add logger when done

* feat: finish statistic jobs (not tested)

* feat: create a separate interval job that gets the current date and create statistics jobs with date

* feat: update logic using dayjs and lodash lib

* feat: update cron jobs logic, move api actions to its service folder

* fix: query event from db and group it based on event_id and tx_id

* fix: just use a single job to query all daily data

* fix: move all queries in daily_stats job into a single Promise all

* fix: move dayjs.extend to after import statements

* fix: remove _start

* fix: only count native token in account_stats job

* feat: add api to sync stats from prev dates

* fix: support case when user just pass startDate to api

* fix: fix whitelist graphql

* fix: update column in account_statistic table, update bignum top_tx_sent

* fix: remove drop index in modify account statistic table

---------

Co-authored-by: AnDQK <doquockhanhan@gmail.com>
Co-authored-by: Phan Anh Tuan <fibonacci998@gmail.com>

ci/config.json.ci

@@ -162,6 +162,16 @@
     "dispatchMilisecond": 1000,
     "batchSizeLimit": 10
   },
+  "dailyStatistics": {
+    "recordsPerCall": 100
+  },
+  "accountStatistics": {
+    "numberOfTopRecords": 10,
+    "dayRange": [3, 15, 30]
+  },
+  "dailyStatsJobs": {
+    "jobPattern": "0 0 0 * * ?"
+  },
   "jobRedecodeTx": {
     "limitRecordGet": 100
   }

config.json

@@ -162,6 +162,16 @@
     "dispatchMilisecond": 1000,
     "batchSizeLimit": 10
   },
+  "dailyStatistics": {
+    "recordsPerCall": 100
+  },
+  "accountStatistics": {
+    "numberOfTopRecords": 10,
+    "dayRange": [3, 15, 30]
+  },
+  "dailyStatsJobs": {
+    "jobPattern": "0 0 0 * * ?"
+  },
   "jobRedecodeTx": {
     "limitRecordGet": 100
   }

migrations/20230704102018_create_table_daily_stats_account_stats.ts

@@ -0,0 +1,26 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.createTable('daily_statistics', (table: any) => {
+    table.increments();
+    table.bigint('daily_txs').index().notNullable();
+    table.bigint('daily_active_addresses').index().notNullable();
+    table.bigint('unique_addresses').index().notNullable();
+    table.timestamp('date').unique().notNullable();
+  });
+  await knex.schema.createTable('account_statistics', (table: any) => {
+    table.increments();
+    table.string('address').index().notNullable();
+    table.bigint('amount_sent').index().notNullable();
+    table.bigint('amount_received').index().notNullable();
+    table.bigint('tx_sent').index().notNullable();
+    table.bigint('gas_used').index().notNullable();
+    table.timestamp('date').index().notNullable();
+    table.unique(['address', 'date']);
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTable('account_statistics');
+  await knex.schema.dropTable('daily_statistics');
+}

migrations/20230817015916_modify_column_table_account_statistic.ts

@@ -0,0 +1,13 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('account_statistics', (table) => {
+    table.integer('tx_sent').alter();
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable('account_statistics', (table) => {
+    table.bigint('tx_sent').alter();
+  });
+}

package.json

@@ -64,6 +64,7 @@
     "bull": "^4.10.2",
     "bullmq": "^3.13.3",
     "cosmjs-types": "^0.6.1",
+    "dayjs": "^1.11.9",
     "dotenv": "^16.0.3",
     "file-type": "^18.4.0",
     "graphql": "^16.6.0",

src/common/constant.ts

@@ -1,6 +1,7 @@
 export const REDIS_KEY = {
   IBC_DENOM: 'ibc_denom',
   DASHBOARD_STATISTICS: 'dashboard_statistics',
+  TOP_ACCOUNTS: 'top_accounts',
 };
 
 export const URL_TYPE_CONSTANTS = {
@@ -62,6 +63,10 @@ export const BULL_JOB_NAME = {
     'job:check-need-create-event-attr-partition',
   JOB_CREATE_EVENT_ATTR_PARTITION: 'job:create-event-attr-partition',
   CRAWL_GENESIS_FEEGRANT: 'crawl:genesis-feegrant',
+  CRAWL_DAILY_STATISTICS: 'crawl:daily-statistics',
+  CRAWL_ACCOUNT_STATISTICS: 'crawl:account-statistics',
+  HANDLE_TOP_ACCOUNTS: 'handle:top-accounts',
+  HANDLE_DAILY_STATS_JOBS: 'handle:daily-stats-jobs',
   REINDEX_CW721_CONTRACT: 'reindex:cw721-contract',
   REINDEX_CW721_HISTORY: 'reindex:cw721-history',
   HANDLE_MIGRATE_CONTRACT: 'handle:migrate-contract',
@@ -225,6 +230,26 @@ export const SERVICE = {
         path: 'v1.ReDecodeTx',
       },
     },
+    DailyStatisticsService: {
+      key: 'DailyStatisticsService',
+      name: 'v1.DailyStatisticsService',
+      CreateSpecificDateJob: {
+        key: 'CreateSpecificDateJob',
+        path: 'v1.DailyStatisticsService.CreateSpecificDateJob',
+      },
+    },
+    AccountStatisticsService: {
+      key: 'AccountStatisticsService',
+      name: 'v1.AccountStatisticsService',
+      CreateSpecificDateJob: {
+        key: 'CreateSpecificDateJob',
+        path: 'v1.AccountStatisticsService.CreateSpecificDateJob',
+      },
+    },
+    DailyStatsJobsService: {
+      key: 'DailyStatsJobsService',
+      name: 'v1.DailyStatsJobsService',
+    },
     Cw20ReindexingService: {
       key: 'Cw20ReindexingService',
       name: 'v1.Cw20ReindexingService',

src/common/utils/request.ts

@@ -8,3 +8,11 @@ export interface IProposalIdParam {
 export interface ITxIdsParam {
   txIds: number[];
 }
+
+export interface IStatisticsParam {
+  date: string;
+}
+
+export interface ICreateSpecificDateJob {
+  date: string;
+}

src/models/account_statistics.ts

@@ -0,0 +1,56 @@
+import BaseModel from './base';
+
+export class AccountStatistics extends BaseModel {
+  address!: string;
+
+  amount_sent!: string;
+
+  amount_received!: string;
+
+  tx_sent!: number;
+
+  gas_used!: string;
+
+  date!: Date;
+
+  static get tableName() {
+    return 'account_statistics';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'address',
+        'amount_sent',
+        'amount_received',
+        'tx_sent',
+        'gas_used',
+        'date',
+      ],
+      properties: {
+        address: { type: 'string' },
+        amount_sent: { type: 'string' },
+        amount_received: { type: 'string' },
+        tx_sent: { type: 'number' },
+        gas_used: { type: 'string' },
+        date: { type: 'string', format: 'date-time' },
+      },
+    };
+  }
+
+  static get relationMappings() {
+    return {};
+  }
+
+  static newAccountStat(address: string, date: string) {
+    return AccountStatistics.fromJson({
+      address,
+      amount_sent: '0',
+      amount_received: '0',
+      tx_sent: 0,
+      gas_used: '0',
+      date,
+    });
+  }
+}

src/models/daily_statistics.ts

@@ -0,0 +1,37 @@
+import BaseModel from './base';
+
+export class DailyStatistics extends BaseModel {
+  daily_txs!: number;
+
+  daily_active_addresses!: number;
+
+  unique_addresses!: number;
+
+  date!: Date;
+
+  static get tableName() {
+    return 'daily_statistics';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'daily_txs',
+        'daily_active_addresses',
+        'unique_addresses',
+        'date',
+      ],
+      properties: {
+        daily_txs: { type: 'number' },
+        daily_active_addresses: { type: 'number' },
+        unique_addresses: { type: 'number' },
+        date: { type: 'string', format: 'date-time' },
+      },
+    };
+  }
+
+  static get relationMappings() {
+    return {};
+  }
+}

src/models/event.ts

@@ -89,6 +89,10 @@ export class Event extends BaseModel {
     REVOKE_FEEGRANT: 'revoke_feegrant',
     USE_FEEGRANT: 'use_feegrant',
     SET_FEEGRANT: 'set_feegrant',
+    COIN_SPENT: 'coin_spent',
+    COIN_RECEIVED: 'coin_received',
+    TX: 'tx',
+    TRANSFER: 'transfer',
     MIGRATE: 'migrate',
   };
 }

src/models/event_attribute.ts

@@ -2,6 +2,7 @@ import { Model } from 'objection';
 import BaseModel from './base';
 // eslint-disable-next-line import/no-cycle
 import { Event } from './event';
+import { Transaction } from './transaction';
 
 export class EventAttribute extends BaseModel {
   event_id!: string;
@@ -52,6 +53,14 @@ export class EventAttribute extends BaseModel {
           to: 'event.id',
         },
       },
+      transaction: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: Transaction,
+        join: {
+          from: 'event_attribute.tx_id',
+          to: 'transaction.id',
+        },
+      },
     };
   }
 
@@ -61,13 +70,10 @@ export class EventAttribute extends BaseModel {
     REDELEGATION_RESPONSES: 'redelegation_responses',
     UNBONDING_RESPONSES: 'unbonding_responses',
     ACTION: 'action',
-    TRANSFER: 'transfer',
     SENDER: 'sender',
     RECEIVER: 'receiver',
     SPENDER: 'spender',
     RECIPIENT: 'recipient',
-    COIN_RECEIVED: 'coin_received',
-    COIN_SPENT: 'coin_spent',
     WITHDRAW_REWARDS: 'withdraw_rewards',
     AMOUNT: 'amount',
     VALIDATOR: 'validator',
@@ -90,5 +96,18 @@ export class EventAttribute extends BaseModel {
     GRANTER: 'granter',
     GRANTEE: 'grantee',
     FROM: 'from',
+    FEE: 'fee',
+    FEE_PAYER: 'fee_payer',
+  };
+
+  static ATTRIBUTE_COMPOSITE_KEY = {
+    COIN_SPENT_SPENDER: 'coin_spent.spender',
+    COIN_RECEIVED_RECEIVER: 'coin_received.receiver',
+    COIN_SPENT_AMOUNT: 'coin_spent.amount',
+    COIN_RECEIVED_AMOUNT: 'coin_received.amount',
+    USE_FEEGRANT_GRANTER: 'use_feegrant.granter',
+    USE_FEEGRANT_GRANTEE: 'use_feegrant.grantee',
+    TX_FEE: 'tx.fee',
+    TX_FEE_PAYER: 'tx.fee_payer',
   };
 }

src/models/index.ts

@@ -22,4 +22,6 @@ export * from './delegator';
 export * from './code_id_verification';
 export * from './feegrant';
 export * from './feegrant_history';
+export * from './daily_statistics';
+export * from './account_statistics';
 export * from './cw20_total_holder_stats';

src/services/api-gateways/api_gateway.service.ts

@@ -21,7 +21,17 @@ import { bullBoardMixin } from '../../mixins/bullBoard/bullBoard.mixin';
         path: '/api',
         autoAliases: true, // allow generate rest info (GET/PUT/POST...) in the services
         mappingPolicy: 'restrict', // allow action called with exact method
-        whitelist: ['v2.dashboard-statistics.*', 'v2.graphql.*'],
+        whitelist: [
+          'v2.graphql.*',
+          'v2.statistics.getDashboardStatisticsByChainId',
+          'v2.statistics.getTopAccountsByChainId',
+        ],
+      },
+      {
+        path: '/admin',
+        autoAliases: true, // allow generate rest info (GET/PUT/POST...) in the services
+        mappingPolicy: 'restrict', // allow action called with exact method
+        whitelist: ['v2.statistics.syncPrevDateStatsByChainId'],
       },
       {
         path: '/admin',