added analytics script

scripts/analytics/.env.example

@@ -0,0 +1,3 @@
+PASSWORD_PROD=
+PASSWORD_DEV=
+API_KEY=

scripts/analytics/.eslintrc.js

@@ -0,0 +1,64 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint', 'import', 'sort-imports-es6-autofix'],
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:import/recommended',
+    'plugin:import/typescript'
+  ],
+  rules: {
+    indent: [2, 2, { SwitchCase: 1 }],
+    quotes: [2, 'single'],
+    semi: [1, 'always'],
+    'no-trailing-spaces': [2],
+    'quote-props': [2, 'as-needed'],
+    'eol-last': [2, 'always'],
+    'object-curly-spacing': [2, 'always'],
+    'comma-dangle': [2, {
+      arrays: 'always-multiline',
+      objects: 'always-multiline',
+      imports: 'always-multiline',
+      exports: 'always-multiline',
+      functions: 'only-multiline',
+    }],
+
+    /* ---------- turn off ---------- */
+    '@typescript-eslint/no-extra-semi': 0,
+    '@typescript-eslint/no-use-before-define': 0,
+    '@typescript-eslint/explicit-member-accessibility': 0,
+    '@typescript-eslint/naming-convention': 0,
+    '@typescript-eslint/no-explicit-any': 0, // any is sometimes unavoidable
+    '@typescript-eslint/consistent-type-definitions': 0, // can use Type and Interface
+    '@typescript-eslint/explicit-function-return-type': 0, // type inference on return type is useful
+    '@typescript-eslint/no-parameter-properties': 0,
+    '@typescript-eslint/typedef': 0,
+    'no-unused-expressions': 0, // short ciucuit if
+    'max-lines': 0,
+    '@typescript-eslint/no-empty-function': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    'sort-imports-es6-autofix/sort-imports-es6': 'warn',
+    '@typescript-eslint/ban-ts-comment': 'off',
+    'no-useless-escape': 'off',
+    '@typescript-eslint/no-non-null-asserted-optional-chain': 'off',
+    'import/no-named-as-default-member': 'off',
+    'import/no-named-as-default': 'off',
+    '@typescript-eslint/no-non-null-assertion': 'off',
+    '@typescript-eslint/no-unused-vars': [
+      'warn',
+      {
+        argsIgnorePattern: '^_',
+        varsIgnorePattern: '^_',
+        caughtErrorsIgnorePattern: '^_'
+      }
+    ]
+  },
+  settings: {
+    'import/resolver': {
+      typescript: {
+        project: 'tsconfig.json'
+      }
+    }
+  }
+};

scripts/analytics/.gitignore

@@ -0,0 +1,74 @@
+# Logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Optional npm cache directory
+.npm
+
+# Optional yarn cache directory
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+.yarn/versions
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# next.js build output
+.next
+
+# OS X temporary files
+.DS_Store
+
+dist
+lib
+*.tsbuildinfo
+
+*.csv

scripts/analytics/README.md

@@ -0,0 +1,8 @@
+# Acala Analytics Scripts
+first fill `.env.example` and `mv .env.example .env`
+
+pull data from postgres and upload to foorprint automatically
+```
+yarn update
+```
+

scripts/analytics/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "acala-analytics-scrips",
+  "version": "1.0.0",
+  "main": "src/index.ts",
+  "license": "MIT",
+  "scripts": {
+    "update": "ts-node src/index.ts"
+  },
+  "devDependencies": {
+    "@types/axios": "^0.14.0",
+    "@types/node": "^20.8.4",
+    "@types/papaparse": "^5",
+    "@types/pg": "^8.10.4",
+    "@typescript-eslint/eslint-plugin": "^6.7.5",
+    "@typescript-eslint/parser": "^6.7.5",
+    "axios": "^1.5.1",
+    "dotenv": "^16.3.1",
+    "envalid": "^8.0.0",
+    "eslint": "^8.51.0",
+    "eslint-import-resolver-typescript": "^3.5.5",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-sort-imports-es6-autofix": "^0.6.0",
+    "papaparse": "^5.4.1",
+    "pg": "^8.11.3",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.2.2"
+  }
+}

scripts/analytics/src/actions/index.ts

@@ -0,0 +1,3 @@
+export * from './pull';
+export * from './transform';
+export * from './upload';

scripts/analytics/src/actions/pull.ts

@@ -0,0 +1,75 @@
+import Papa from 'papaparse';
+import { Client, ClientConfig } from 'pg';
+import dotenv from 'dotenv';
+import fs from 'fs';
+
+dotenv.config();
+
+interface QueryTarget {
+  schema: string,
+  tables: string[],
+  filenames: string[],
+}
+type QueryParams = ClientConfig & QueryTarget;
+
+const getAllTables = async (client: Client, schema: string) => {
+  console.log(`querying all tables under schema ${schema} ...`);
+
+  const res = await client.query(`
+        SELECT table_name 
+        FROM information_schema.tables 
+        WHERE table_schema = $1
+    `, [schema]);
+
+  return res.rows.map(row => row.table_name);
+};
+
+export const pullDataFromDb = async ({
+  host,
+  port,
+  database,
+  user,
+  password,
+  schema,
+  tables,
+  filenames,
+}: QueryParams) => {
+  if (tables.length !== filenames.length) {
+    throw new Error('tables and filenames should have the same length');
+  }
+
+  const client = new Client({
+    host: host,
+    port: port,
+    database: database,
+    user: user,
+    password: password,
+  });
+
+  const savedFiles = [];
+  try {
+    console.log('connecting to db ...');
+    await client.connect();
+    console.log('db connected!');
+
+    const tableNames = tables ?? await getAllTables(client, schema);
+
+    for (const [i, table] of tableNames.entries()) {
+      const targetFile = filenames[i] ?? `${table}.csv`;
+      const dataRes = await client.query(`SELECT * FROM "${schema}"."${table}"`);
+
+      const csv = Papa.unparse(dataRes.rows);
+      fs.writeFileSync(targetFile, csv);
+
+      savedFiles.push(targetFile);
+      console.log(`saved [${schema}.${table}] data to [${targetFile}]`);
+    }
+
+  } catch (err) {
+    console.error('Error fetching data:', err);
+  } finally {
+    await client.end();
+  }
+
+  return savedFiles;
+};

scripts/analytics/src/actions/transform.ts

@@ -0,0 +1,110 @@
+import { FP_DATA_TYPE } from '../consts';
+import { readCSV, writeCSV } from '../utils';
+
+interface StakeTxRaw {
+  id: string;
+  from: string;
+  tx_hash: string;
+  block_number: string;
+  timestamp: string;
+  pool_id: string;
+  amount: string;
+}
+
+interface StakeTx {
+  from: string;
+  tx_hash: string;
+  block_number: number;
+  timestamp: string;
+  pool_id: number;
+  amount: number;
+}
+
+// truncate *last* x percent of data
+const _truncate = async (csvData: any[], percent: number) => {
+  if (percent <= 0 || percent > 100) {
+    throw new Error('invalid truncate percent');
+  }
+
+  const rowCount = csvData.length;
+  const rowsToExtract = Math.ceil(rowCount * (percent / 100));
+  return csvData.slice(-rowsToExtract);
+};
+
+const pickColumns = (csvData: any[], columns: string[]) => csvData.map(d => 
+  columns.reduce((acc, col) => ({ ...acc, [col]: d[col] }), {}),
+);
+
+// this shape is compatible with Footprint and dune
+// TODO: there should be a lib that does this?
+const formatDate = (input: string): string => {
+  const date = new Date(input);
+
+  const YYYY = date.getFullYear();
+  const MM = String(date.getMonth() + 1).padStart(2, '0'); // Months are 0-based, so we add 1
+  const DD = String(date.getDate()).padStart(2, '0');
+  const HH = String(date.getHours()).padStart(2, '0');
+  const mm = String(date.getMinutes()).padStart(2, '0');
+  const ss = String(date.getSeconds()).padStart(2, '0');
+
+  return `${YYYY}-${MM}-${DD} ${HH}:${mm}:${ss}`;
+};
+
+const toSimpleTimestamp = (csvData: any[]) => csvData.map(rowData => ({
+  ...rowData,
+  timestamp: formatDate(rowData.timestamp),
+}));
+
+export async function transformCSV(filename: string): Promise<void> {
+  console.log(`transforming ${filename} ...`);
+  const rawData = await readCSV(filename);
+  const data = toSimpleTimestamp(pickColumns(rawData, ['timestamp', 'pool_id', 'amount', 'from', 'type']));
+
+  await writeCSV(filename, data);
+  console.log('transformation finished!');
+}
+
+export const toFPCompatible = (data: any[], type: FP_DATA_TYPE) => {
+  if (type === FP_DATA_TYPE.EuphratesStake) {
+    return data.map<StakeTx>((row: StakeTxRaw) => ({
+      block_number: parseInt(row.block_number, 10),
+      timestamp: row.timestamp,
+      from: row.from,
+      tx_hash: row.tx_hash,
+      pool_id: parseInt(row.pool_id, 10),
+      amount: parseInt(row.amount, 10),
+    }));
+  } if (type === FP_DATA_TYPE.AcalaLogs) {
+    return data.map((row: any) => ({
+      block_number: parseInt(row.block_number, 10),
+      timestamp: row.timestamp,
+      block_hash: row.block_hash,
+      transaction_index: parseInt(row.transaction_index, 10),
+      log_index: parseInt(row.log_index, 10),
+      address: row.address,
+      data: row.data,
+      topics: row.topics,
+      transaction_hash: row.transaction_hash,
+    }));
+  } if (type === FP_DATA_TYPE.AcalaReceipts) {
+    return data.map((row: any) => ({
+      block_number: parseInt(row.block_number, 10),
+      timestamp: row.timestamp,
+      from: row.from,
+      to: row.to,
+      block_hash: row.block_hash,
+      transaction_index: parseInt(row.transaction_index, 10),
+      effective_gas_price: parseInt(row.effective_gas_price, 10),
+      cumulative_gas_used: parseInt(row.cumulative_gas_used, 10),
+      type: parseInt(row.type, 10),
+      status: parseInt(row.status, 10),
+      gas_used: parseInt(row.gas_used, 10),
+      contract_address: row.contract_address,
+      transaction_hash: row.transaction_hash,
+      logs_bloom: row.logs_bloom,
+      exit_reason: row.exit_reason,
+    }));
+  }
+
+  throw new Error(`<toFPCompatible> invalid type: ${type}`);
+};