Fix/missed blocks not working 1208

.github/workflows/deploy-libre-testnet.yaml

@@ -56,7 +56,7 @@ jobs:
           REACT_APP_SYNC_TOLERANCE_INTERVAL: 180000
           REACT_APP_TOKEN_SYMBOL: 'LIBRE'
           REACT_APP_NETWORK_URL:  '[{"label":"EOS","value":"https://eos.antelope.tools","mainnet":true,"pair":"eos","icon":"eos","order":1},{"label":"Proton","value":"https://proton.antelope.tools","mainnet":true,"pair":"proton","icon":"proton","order":2},{"label":"WAX","value":"https://wax.antelope.tools","mainnet":true,"pair":"wax","icon":"wax","order":3},{"label":"Telos","value":"https://telos.antelope.tools","mainnet":true,"pair":"telos","icon":"telos","order":4},{"label":"Libre","value":"https://libre.antelope.tools","mainnet":true,"pair":"libre","icon":"libre","order":5},{"label":"LACChain EOSIO","value":"https://lacchain.antelope.tools","mainnet":true,"pair":null,"icon":"lacchain","order":6},{"label":"Jungle4 Testnet","value":"https://jungle.antelope.tools","mainnet":false,"pair":"eos","icon":"jungle","order":1},{"label":"Proton Testnet","value":"https://proton-testnet.antelope.tools","mainnet":false,"pair":"proton","icon":"proton","order":2},{"label":"WAX Testnet","value":"https://wax-testnet.antelope.tools","mainnet":false,"pair":"wax","icon":"wax","order":3},{"label":"Telos Testnet","value":"https://telos-testnet.antelope.tools","mainnet":false,"pair":"telos","icon":"telos","order":4},{"label":"Libre Testnet","value":"https://libre-testnet.antelope.tools","mainnet":false,"pair":"libre","icon":"libre","order":5},{"label":"Ultra Testnet","value":"https://ultra-testnet.antelope.tools","mainnet":false,"pair":"ultra","icon":"ultra","order":6}]'
-          REACT_APP_DISABLED_MENU_ITEMS: '["/missed-blocks"]'
+          REACT_APP_DISABLED_MENU_ITEMS: '[]'
           REACT_APP_BLOCK_EXPLORER_URL: 'https://testnet.libre.org/v2/explore/transaction/(transaction)'
           REACT_APP_STATE_HISTORY_ENABLED: 'true'
           REACT_APP_GOOGLE_ANALITIC_PAGE_ID: 'G-E6Y0EC9FT8'
@@ -83,7 +83,7 @@ jobs:
           HAPI_EOS_API_NETWORK_NAME: libre
           HAPI_EOS_API_ENDPOINTS: '["https://libre-testnet.edenia.cloud","https://api.testnet.libre.cryptobloks.io","https://libre-testnet.eosphere.io"]'
           HAPI_EOS_STATE_HISTORY_PLUGIN_ENDPOINT: 'ws://api-node.libre-testnet:8080'
-          HAPI_EOS_MISSED_BLOCKS_ENABLED: 'false'
+          HAPI_EOS_MISSED_BLOCKS_ENABLED: 'true'
           HAPI_EOS_BLOCK_HISTORY_DAYS: 90
           HAPI_EOS_MAX_CPU_BLOCK: 250000
           HAPI_EOS_MAX_NET_BLOCK: 1048576

hapi/src/services/missed-blocks.service.js

@@ -5,13 +5,14 @@ const {
   sequelizeUtil,
   hasuraUtil,
   sleepFor,
-  getGranularityFromRange
+  getGranularityFromRange,
+  eosUtil
 } = require('../utils')
 
 const setScheduleHistory = items => {
   const mutation = `
     mutation ($items: [schedule_history_insert_input!]!) {
-      insert_schedule_history(objects: $items, on_conflict: {constraint: schedule_history_version_key, update_columns: [first_block_at, last_block_at, first_block, last_block, producers, current, round_interval]}) {
+      insert_schedule_history(objects: $items, on_conflict: {constraint: schedule_history_version_key, update_columns: [first_block_at, first_block, producers, round_interval]}) {
         affected_rows
       }
     }
@@ -36,9 +37,7 @@ const setScheduleByDemux = async (state, payload) => {
     SELECT
       schedule_version as version,
       min(timestamp) as first_block_at,
-      max(timestamp) as last_block_at,
       min(block_num) as first_block,
-      max(block_num) as last_block
     FROM
       block_history
     WHERE schedule_version = ${currentVersion + 1}
@@ -56,16 +55,50 @@ const setScheduleByDemux = async (state, payload) => {
   await setScheduleHistory(schedules)
 }
 
-const getLastRoundInfo = async () => {
-  const stats = await statsService.getStats()
+const getScheduleFirstBlock = async version => {
+  const [rows] = await sequelizeUtil.query(`
+  SELECT 
+    block_history.block_num, timestamp, producer
+  FROM 
+    block_history
+  INNER JOIN(
+    SELECT
+      min(block_num) as block_num
+    FROM
+      block_history
+    WHERE 
+      schedule_version = ${version}
+  ) as first_block
+  ON block_history.block_num = first_block.block_num`)
 
-  // if there is no stats we should try later
-  if (!stats) {
-    return null
+  return rows[0]
+}
+
+const syncCurrentSchedule = async () => {
+  const current = await getCurrentVersion()
+  const { active: schedule } = await eosUtil.getProducerSchedule()
+  const producers = schedule.producers.map(producer => producer.producer_name)
+
+  if(schedule.version > current) {
+    const firstBlock = await getScheduleFirstBlock(schedule.version)
+
+    if (!firstBlock) return
+
+    await setScheduleHistory({
+      version: schedule.version,
+      first_block: firstBlock.block_num,
+      first_block_at: firstBlock.timestamp,
+      producers, 
+      round_interval: producers.length * 6
+    })
   }
+}
+
+const getLastRoundInfo = async () => {
+  const stats = await statsService.getStats()
 
   // if there is no last block we should try later
-  if (!stats.last_block_at) {
+  if (!stats?.last_block_at) {
     return null
   }
 
@@ -76,10 +109,10 @@ const getLastRoundInfo = async () => {
     }
   }
 
-  // if there is no previous round we should start from round 0 and version 0
+  // if there is no previous round we should start from the first available schedule
   const query = `
     query {
-      schedule_history (where: {version: {_eq: 0}}) {
+      schedule_history (limit: 1, order_by: {version: asc}) {
         schedule: version,
         first_block_at,
         interval: round_interval,
@@ -126,13 +159,14 @@ const getBlocksInRange = async (start, end) => {
   }))
 }
 
-const getScheduleByVersion = async version => {
+const getNextScheduleByVersion = async version => {
   const query = `
     query {
-      schedule_history(where: {version: {_eq: ${version}}}, limit: 1) {
+      schedule_history(where: {version: {_gte: ${version}}}, order_by: {version: asc}, limit: 1) {
         version
         producers
         round_interval
+        first_block_at
       }
     }  
   `
@@ -172,35 +206,36 @@ const syncMissedBlocks = async () => {
   // if the diference between the
   // last block time and the end time
   // is less than the round interval
-  // then wait until the round end
+  // then wait for block history synchronization
   if (
     moment(lastRound.last_block_at).diff(end, 'seconds') < lastRound.interval
   ) {
-    await sleepFor(60)
+    await sleepFor(10)
     syncMissedBlocks()
 
     return
   }
 
   const blocks = await getBlocksInRange(start, end)
 
-  // if the first block comes from different schedule_version
-  // then it's the end of the version in use
-  if (blocks.length > 0 && blocks[0].schedule_version !== lastRound.schedule) {
-    const newSchedule = await getScheduleByVersion(lastRound.schedule + 1)
+  if (!blocks.length || blocks.length > 0 && blocks[0].schedule_version !== lastRound.schedule) {
+    const newSchedule = await getNextScheduleByVersion(
+      lastRound.schedule + 1
+    )
 
-    // schedule version no yet in the history
+    // schedule version no yet in the DB
     if (!newSchedule) {
       await sleepFor(60)
       syncMissedBlocks()
 
       return
     }
 
-    lastRound.schedule += 1
+    lastRound.schedule = newSchedule.version
     lastRound.number = 0
     lastRound.interval = newSchedule.round_interval
     lastRound.producers = newSchedule.producers
+    lastRound.completed_at = newSchedule.first_block_at
 
     await statsService.udpateStats({ last_round: lastRound })
     syncMissedBlocks()
@@ -210,22 +245,21 @@ const syncMissedBlocks = async () => {
 
   lastRound.number += 1
   lastRound.completed_at = end.toISOString()
-  const roundHistory = lastRound.producers.map(producer => {
-    const producerBlocks = blocks.filter(block => block.producer === producer)
+  const roundHistory = lastRound.producers.map((producer) => {
+    const producerBlocks = blocks.filter((block) => block.producer === producer)
 
     return {
       schedule: lastRound.schedule,
       number: lastRound.number,
       account: producer,
-      started_at: start.toISOString(),
       completed_at: end.toISOString(),
-      missed_blocks: 12 - producerBlocks.length,
-      produced_blocks: producerBlocks.length
+      missed_blocks: 12 - producerBlocks.length
     }
   })
 
   await addRoundHistory(roundHistory)
   await statsService.udpateStats({ last_round: lastRound })
+
   syncMissedBlocks()
 }
 
@@ -244,8 +278,8 @@ const getMissedBlocks = async (range = '3 Hours') => {
       interval.value as datetime,
       round_history.account,
       sum(round_history.missed_blocks) as missed,
-      sum(round_history.produced_blocks) as produced,
-      sum(round_history.missed_blocks)+sum(round_history.produced_blocks) as scheduled
+      count(1) * 12 - sum(round_history.missed_blocks) as produced,
+      count(1) * 12 as scheduled
     FROM 
       interval
     INNER JOIN 
@@ -261,6 +295,7 @@ const getMissedBlocks = async (range = '3 Hours') => {
 
 module.exports = {
   syncMissedBlocks,
+  syncCurrentSchedule,
   getMissedBlocks,
   setScheduleByDemux
 }

hapi/src/services/state-history-plugin.service.js

@@ -133,10 +133,10 @@ const handleBlocksResult = async (data) => {
 
     if (blocksData.length === 50) {
       await saveBlocks(blocksData)
+      await statsService.udpateStats({ last_block_at: block.timestamp })
       blocksData = []
     }
 
-    await statsService.udpateStats({ last_block_at: block.timestamp })
     send(
       serialize('request', ['get_blocks_ack_request_v0', { num_messages: 1 }])
     )
@@ -156,6 +156,9 @@ const cleanOldBlocks = async () => {
       delete_block_history (where: {timestamp: {_lt: $date}}) {
         affected_rows
       }
+      delete_round_history (where: {completed_at: {_lt: $date}}) {
+        affected_rows
+      }
     }
   `
 

hapi/src/services/stats.service.js

@@ -9,7 +9,7 @@ const STAT_ID = 'bceb5b75-6cb9-45af-9735-5389e0664847'
 const _getScheduleHystory = async () => {
   const query = `
     query {
-      schedule_history (where: {version: {_eq: 0}}) {
+      schedule_history (limit: 1, order_by: {version: asc}) {
         first_block_at
       }
     }
@@ -33,7 +33,7 @@ const _getMissedBlock = async (start, end) => {
   const [rows] = await sequelizeUtil.query(`
     SELECT account, sum(missed_blocks)
     FROM round_history
-    WHERE updated_at
+    WHERE completed_at
     BETWEEN '${start}'::timestamp AND '${end}'::timestamp
     GROUP BY account
   `)
@@ -153,6 +153,7 @@ const getStats = async () => {
         last_block_at
         tps_all_time_high
         missed_blocks
+        missed_blocks_checked_at
         updated_at
         created_at
       }
@@ -173,9 +174,9 @@ const getCurrentMissedBlock = async () => {
 
   if (!stats) return
 
-  if (stats.missed_blocks) {
+  if (stats.missed_blocks_checked_at && stats.last_round) {
     data = stats.missed_blocks
-    lastBlockAt = stats.last_block_at
+    lastBlockAt = stats.last_round.completed_at
   } else {
     const scheduleHistoryInfo = await _getScheduleHystory()
 
@@ -189,9 +190,8 @@ const getCurrentMissedBlock = async () => {
 
     if (!rowsInitial.length) {
       await udpateStats({
-        missed_blocks: {
-          checked_at: end.toISOString()
-        }
+        missed_blocks: {},
+        missed_blocks_checked_at: end.toISOString()
       })
 
       getCurrentMissedBlock()
@@ -200,7 +200,7 @@ const getCurrentMissedBlock = async () => {
     }
   }
 
-  start = moment(data.checked_at).add(1, 'second')
+  start = moment(stats.missed_blocks_checked_at).add(1, 'second')
   end = moment(start).add(59, 'seconds')
 
   if (_checkDateGap(lastBlockAt, end)) {
@@ -214,37 +214,28 @@ const getCurrentMissedBlock = async () => {
 
   if (!rows.length) {
     await udpateStats({
-      missed_blocks: {
-        ...data,
-        checked_at: end.toISOString()
-      }
+      missed_blocks: data,
+      missed_blocks_checked_at: end.toISOString()
     })
 
     getCurrentMissedBlock()
 
     return
   }
 
-  let newData = data
+  const newData = data
 
   rows.forEach((element) => {
-    if (newData[element.account]) {
-      newData = {
-        ...newData,
-        [element.account]: `${
-          parseInt(newData[element.account]) + parseInt(element.sum)
-        }`
-      }
-    } else {
-      newData = { ...newData, [element.account]: element.sum }
+    const sum = parseInt(element.sum)
+
+    if (sum > 0) {
+      newData[element.account] = sum + (parseInt(newData[element.account]) || 0)
     }
   })
 
   await udpateStats({
-    missed_blocks: {
-      ...newData,
-      checked_at: end.toISOString()
-    }
+    missed_blocks: newData,
+    missed_blocks_checked_at: end.toISOString()
   })
 
   getCurrentMissedBlock()

hapi/src/workers/producers.worker.js

@@ -4,8 +4,7 @@ const {
   producerService,
   settingService,
   stateHistoryPluginService,
-  statsService,
-  demuxService
+  statsService
 } = require('../services')
 const { workersConfig, hasuraConfig, eosConfig } = require('../config')
 const { axiosUtil, sleepFor } = require('../utils')
@@ -72,7 +71,7 @@ const start = async () => {
     if (eosConfig.missedBlocksServiceEnabled) {
       run('SYNC MISSED BLOCKS', missedBlocksService.syncMissedBlocks)
       run('SYNC MISSED BLOCKS PER PRODUCER', statsService.getCurrentMissedBlock)
-      run('SYNC SCHEDULE HISTORY', demuxService.init)
+      run('SYNC SCHEDULE HISTORY', missedBlocksService.syncCurrentSchedule, 60)
     }
 
     run('SYNC TPS', statsService.syncTPSAllTimeHigh)

hasura/migrations/default/1689956887100_alter_table_public_stat_add_column_missed_blocks_checked_at/down.sql

@@ -0,0 +1,4 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- alter table "public"."stat" add column "missed_blocks_checked_at" timestamp with time zone
+--  null;

hasura/migrations/default/1689956887100_alter_table_public_stat_add_column_missed_blocks_checked_at/up.sql

@@ -0,0 +1,2 @@
+alter table "public"."stat" add column "missed_blocks_checked_at" timestamp with time zone
+ null;

hasura/migrations/default/1689962393656_drop_columns_round_and_schedule_history/down.sql

@@ -0,0 +1,12 @@
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- alter table "public"."round_history"
+-- drop column "updated_at",
+-- drop column "started_at",
+-- drop column "produced_blocks";
+--
+-- alter table "public"."schedule_history"
+-- drop column "updated_at",
+-- drop column "current",
+-- drop column "last_block",
+-- drop column "last_block_at";