feat: ingestion and querying for new PoX-2 events (#1403)

* test: Rosetta PoX-2 rewards and unlock events

* feat: progress on parsing and writing PoX2 events to postgres

* feat: sql insert for pox2 event data

* feat: reading pox2 event data from db, sql constraints on named pox event types

* feat: db-controller and initial GET route for reading & parsing pox2 events

* chore: progress on e2e testing for pox-2 delegation functionality

* chore: e2e testing for pox-2 delegation commit operation

* chore: e2e testing for pox-2 delegate-stack-increase

* chore: e2e testing for pox-2 delegate-stack-extend

* chore: e2e testing for pox-2 handle-unlock (missed reward slot)

* chore: lint fix, possible test fix

* chore: validation tests for each of the pox2-event payloads

* chore: potential test fixes

* chore: test fixes num 2

* chore: additional flaky test fixes

* feat: use pox2-events table to calculate account STX locked amount in /extended/v1/address balance endpoints

* chore: Rosetta integration tests for using segwit address in stacking operation

* test: ensure stacks RPC reports expected locked balance after Rosetta lock op

* chore: bump to latest pox2-events stacks-node image

* chore: move pox2-event parsing code into separate file

* ci: bump stacks2.1 test suite time to account for longer block mining interval from latest stacks-node changes

* chore: speed up tests with better "wait for pox cycle" logic

* ci: test fixes

* fix: adjustments to `handle-unlock` processing

* feat: add support for latest additions to pox2: `stack-aggregation-commit-indexed` and `stack-aggregation-increase`

* chore: integration tests for `stack-aggregation-commit-indexed` and `stack-aggregation-increase` pox2 event handling

* feat: store pox_v1_unlock_height as singleton in pg

* chore: initial setup for stacks-2.1 transition test suite

* chore: fix pox_state migrations

* feat: make account balance drop pox1-locked state after pox_v1_unlock_height is reached

* fix: rosetta block unlock events were limited to one account

* fix: off-by-one bug causing accounts to be reported as unlocked one block too early

* feat: add burn_block_height to db chainTip and /extended/status

* chore: fix datastore-tests from new status endpoint field

* chore: fix datastore-tests from new status endpoint field, second pass

* fix: adjust pox2_events query to handle special case with `handle-unlock`

* feat: add `contract_name` processing for the `stx_lock_event`, see stacks-network/stacks-core#3413

* chore: bump to latest stacks-network/stacks-core#3415

* chore: add account balance test for consistency between RPC and API balances throughout pox transition

* ci: matrix for stacks2.1-transition tests

* chore: start of Rosetta pox transition period tests

* chore: test Rosetta account balance endpoint in pox-2 stacking test suite

* ci: use unique account key for pox-2 rosetta test suite

* chore: test for pox_v1_unlock_height in Rosetta pox transition tests

* chore: refactor giant pox-2 test file into multiple files, run separately in ci

* feat: generate synthetic Rosetta stx unlock operations for the pox_v1_unlock_height block

* feat: generate Rosetta unlock operations from pox2 auto-unlock events

* fix: store locked balance from `handle-unlock` event rather than resetting to zero

* chore: remove todos

* chore: remove todos

* chore: remove no-longer correct sanity check

* chore: address PR feedback, composite index for pox2_events ordering

Author: zone117x

.github/workflows/ci.yml

@@ -346,6 +346,16 @@ jobs:
           parallel: true
 
   test-2_1:
+    strategy:
+      matrix:
+        suite: [
+          pox-2-auto-unlock,
+          pox-2-btc-address-formats,
+          pox-2-delegate-aggregation,
+          pox-2-delegate-stacking,
+          pox-2-stack-extend-increase,
+          pox-2-rosetta
+        ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -382,7 +392,7 @@ jobs:
           npm run devenv:logs:2.1 -- --no-color &> docker-compose-logs.txt &
 
       - name: Run tests
-        run: npm run test:2.1
+        run: npm run test:2.1 -- --testPathPattern "${{ matrix.suite }}"
 
       - name: Print integration environment logs
         run: cat docker-compose-logs.txt
@@ -404,6 +414,72 @@ jobs:
           flag-name: run-${{ github.job }}
           parallel: true
 
+  test-2_1-transition:
+    strategy:
+      matrix:
+        suite: [
+          Stacking on pox-2 after pox-1 force unlock,
+          Consistent API and RPC account balances through pox transitions,
+          Rosetta - Stacks 2.1 transition tests,
+        ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: |
+            ~/.npm
+            **/node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Install deps
+        run: npm ci --audit=false
+
+      - name: Setup env vars
+        run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0" >> $GITHUB_ENV
+
+      - name: Setup integration environment
+        run: |
+          sudo ufw disable
+          npm run devenv:deploy:2.1-transition -- -d
+          npm run devenv:logs:2.1-transition -- --no-color &> docker-compose-logs.txt &
+
+      - name: Run tests
+        run: npm run test:2.1-transition -- -t "${{ matrix.suite }}"
+
+      - name: Print integration environment logs
+        run: cat docker-compose-logs.txt
+        if: failure()
+
+      - name: Teardown integration environment
+        run: npm run devenv:stop:2.1-transition
+        if: always()
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+        if: always()
+
+      - name: Upload coverage to Coveralls
+        uses: coverallsapp/github-action@master
+        if: always()
+        with:
+          github-token: ${{ secrets.github_token }}
+          flag-name: run-${{ github.job }}
+          parallel: true
+
   test-rosetta:
     runs-on: ubuntu-latest
     steps:
@@ -647,6 +723,7 @@ jobs:
     needs:
       - test
       - test-2_1
+      - test-2_1-transition
       - test-bns
       - test-rosetta
       - test-rosetta-cli-construction
@@ -666,6 +743,7 @@ jobs:
       - lint-docs
       - test
       - test-2_1
+      - test-2_1-transition
       - test-bns
       - test-rosetta
       - test-rosetta-cli-construction

.vscode/launch.json

@@ -227,7 +227,28 @@
       "outputCapture": "std",
       "console": "integratedTerminal",
       "preLaunchTask": "deploy:krypton",
-      "postDebugTask": "stop:krypton"
+      "postDebugTask": "stop:krypton",
+      "smartStep": false,
+      "sourceMaps": true,
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Jest: 2.1 transition",
+      "program": "${workspaceFolder}/node_modules/.bin/jest",
+      "args": [
+        "--testTimeout=3600000",
+        "--runInBand",
+        "--no-cache",
+        "--config",
+        "${workspaceRoot}/tests/jest.config.2.1-transition.js",
+      ],
+      "outputCapture": "std",
+      "console": "integratedTerminal",
+      "preLaunchTask": "deploy:krypton-2.1-transition",
+      "postDebugTask": "stop:krypton-2.1-transition",
+      "smartStep": false,
+      "sourceMaps": true,
     },
     {
       "type": "node",

docker/docker-compose.dev.stacks-krypton-2.1-transition.yml

@@ -1,15 +1,15 @@
 version: '3.7'
 services:
   stacks-blockchain:
-    image: "zone117x/stacks-api-e2e:stacks2.1-transition-a7d904e"
+    image: "zone117x/stacks-api-e2e:stacks2.1-transition-feat-segwit-events-8fb6fad"
     ports:
       - "18443:18443" # bitcoin regtest JSON-RPC interface
       - "18444:18444" # bitcoin regtest p2p
       - "20443:20443" # stacks-node RPC interface
       - "20444:20444" # stacks-node p2p
     environment:
-      MINE_INTERVAL: 2s
-      REWARD_RECIPIENT: STQM73RQC4EX0A07KWG1J5ECZJYBZS4SJ4ERC6WN
+      MINE_INTERVAL: 0.1s
+      # REWARD_RECIPIENT: STQM73RQC4EX0A07KWG1J5ECZJYBZS4SJ4ERC6WN
       STACKS_EVENT_OBSERVER: host.docker.internal:3700
       # STACKS_LOG_TRACE: 1
       # STACKS_LOG_DEBUG: 1

docker/docker-compose.dev.stacks-krypton.yml

@@ -1,7 +1,7 @@
 version: '3.7'
 services:
   stacks-blockchain:
-    image: "zone117x/stacks-api-e2e:stacks2.1-a7d904e"
+    image: "zone117x/stacks-api-e2e:stacks2.1-feat-segwit-events-8fb6fad"
     ports:
       - "18443:18443" # bitcoin regtest JSON-RPC interface
       - "18444:18444" # bitcoin regtest p2p

docs/api/info/get-status.schema.json

@@ -15,6 +15,10 @@
       "type": "string",
       "description": "the current server status"
     },
+    "pox_v1_unlock_height": {
+      "type": "integer",
+      "nullable": true
+    },
     "chain_tip": {
       "$ref": "../../entities/info/chain-tip.schema.json"
     }

docs/entities/info/chain-tip.schema.json

@@ -6,7 +6,8 @@
   "required": [
     "block_height",
     "block_hash",
-    "index_block_hash"
+    "index_block_hash",
+    "burn_block_height"
   ],
   "properties": {
     "block_height": {
@@ -28,6 +29,10 @@
     "microblock_sequence": {
       "type": "integer",
       "description": "the current microblock sequence number"
+    },
+    "burn_block_height": {
+      "type": "integer",
+      "description": "the current burn chain block height"
     }
   }
 }

docs/generated.d.ts

@@ -1609,6 +1609,7 @@ export interface ServerStatusResponse {
    * the current server status
    */
   status: string;
+  pox_v1_unlock_height?: number;
   chain_tip?: ChainTip;
 }
 /**
@@ -1635,6 +1636,10 @@ export interface ChainTip {
    * the current microblock sequence number
    */
   microblock_sequence?: number;
+  /**
+   * the current burn chain block height
+   */
+  burn_block_height: number;
 }
 /**
  * GET request that returns network target block times

migrations/1605100253938_stx_lock_events.js

@@ -57,6 +57,10 @@ exports.up = pgm => {
       type: 'string',
       notNull: true,
     },
+    contract_name: {
+      type: 'string',
+      notNull: true,
+    }
   });
 
   pgm.createIndex('stx_lock_events', [
@@ -71,4 +75,5 @@ exports.up = pgm => {
   pgm.createIndex('stx_lock_events', 'locked_address', { method: 'hash' });
   pgm.createIndex('stx_lock_events', [{ name: 'block_height', sort: 'DESC' }]);
   pgm.createIndex('stx_lock_events', [{ name: 'unlock_height', sort: 'DESC' }]);
+  pgm.createIndex('stx_lock_events', 'contract_name');
 }

migrations/1643755236533_chain_tip.js

@@ -2,7 +2,7 @@
 exports.up = pgm => {
   pgm.createMaterializedView('chain_tip', {}, `
     WITH block_tip AS (
-      SELECT block_height, block_hash, index_block_hash
+      SELECT block_height, block_hash, index_block_hash, burn_block_height
         FROM blocks
         WHERE block_height = (SELECT MAX(block_height) FROM blocks WHERE canonical = TRUE)
     ),