feat: added consensus types and consensus column to miniblocks table (#…

…490)

To execute consensus on miniblocks we need to store extra data per
miniblock.

core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE miniblocks
+    DROP COLUMN IF EXISTS consensus;

core/lib/dal/migrations/20231102144901_add_consensus_fields_for_miniblocks.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE miniblocks
+    ADD COLUMN consensus JSONB NULL;

core/lib/dal/sqlx-data.json

@@ -4410,98 +4410,6 @@
     },
     "query": "UPDATE contract_verification_requests SET status = 'failed', updated_at = now(), error = $2, compilation_errors = $3, panic_message = $4 WHERE id = $1"
   },
-  "5190fad25f0c476380af4013761d42ae97dbd55f87e38ceec33f8e148c5cbb14": {
-    "describe": {
-      "columns": [
-        {
-          "name": "number",
-          "ordinal": 0,
-          "type_info": "Int8"
-        },
-        {
-          "name": "l1_batch_number!",
-          "ordinal": 1,
-          "type_info": "Int8"
-        },
-        {
-          "name": "last_batch_miniblock?",
-          "ordinal": 2,
-          "type_info": "Int8"
-        },
-        {
-          "name": "timestamp",
-          "ordinal": 3,
-          "type_info": "Int8"
-        },
-        {
-          "name": "root_hash?",
-          "ordinal": 4,
-          "type_info": "Bytea"
-        },
-        {
-          "name": "l1_gas_price",
-          "ordinal": 5,
-          "type_info": "Int8"
-        },
-        {
-          "name": "l2_fair_gas_price",
-          "ordinal": 6,
-          "type_info": "Int8"
-        },
-        {
-          "name": "bootloader_code_hash",
-          "ordinal": 7,
-          "type_info": "Bytea"
-        },
-        {
-          "name": "default_aa_code_hash",
-          "ordinal": 8,
-          "type_info": "Bytea"
-        },
-        {
-          "name": "virtual_blocks",
-          "ordinal": 9,
-          "type_info": "Int8"
-        },
-        {
-          "name": "hash",
-          "ordinal": 10,
-          "type_info": "Bytea"
-        },
-        {
-          "name": "protocol_version!",
-          "ordinal": 11,
-          "type_info": "Int4"
-        },
-        {
-          "name": "fee_account_address?",
-          "ordinal": 12,
-          "type_info": "Bytea"
-        }
-      ],
-      "nullable": [
-        false,
-        null,
-        null,
-        false,
-        false,
-        false,
-        false,
-        true,
-        true,
-        false,
-        false,
-        true,
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Int8"
-        ]
-      }
-    },
-    "query": "\n                SELECT miniblocks.number,\n                    COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\",\n                    (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\",\n                    miniblocks.timestamp,\n                    miniblocks.hash as \"root_hash?\",\n                    miniblocks.l1_gas_price,\n                    miniblocks.l2_fair_gas_price,\n                    miniblocks.bootloader_code_hash,\n                    miniblocks.default_aa_code_hash,\n                    miniblocks.virtual_blocks,\n                    miniblocks.hash,\n                    miniblocks.protocol_version as \"protocol_version!\",\n                    l1_batches.fee_account_address as \"fee_account_address?\"\n                FROM miniblocks\n                LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number\n                WHERE miniblocks.number = $1\n            "
-  },
   "51cb712685991ffd600dce59f5ed8b5a1bfce8feed46ebd02471c43802e6e65a": {
     "describe": {
       "columns": [
@@ -6178,6 +6086,104 @@
     },
     "query": "SELECT * FROM call_traces WHERE tx_hash IN (SELECT hash FROM transactions WHERE miniblock_number = $1)"
   },
+  "7947dd8e7d6c138146f7ebe6b1e89fcd494b2679ac4e9fcff6aa2b2944aeed50": {
+    "describe": {
+      "columns": [
+        {
+          "name": "number",
+          "ordinal": 0,
+          "type_info": "Int8"
+        },
+        {
+          "name": "l1_batch_number!",
+          "ordinal": 1,
+          "type_info": "Int8"
+        },
+        {
+          "name": "last_batch_miniblock?",
+          "ordinal": 2,
+          "type_info": "Int8"
+        },
+        {
+          "name": "timestamp",
+          "ordinal": 3,
+          "type_info": "Int8"
+        },
+        {
+          "name": "root_hash?",
+          "ordinal": 4,
+          "type_info": "Bytea"
+        },
+        {
+          "name": "l1_gas_price",
+          "ordinal": 5,
+          "type_info": "Int8"
+        },
+        {
+          "name": "l2_fair_gas_price",
+          "ordinal": 6,
+          "type_info": "Int8"
+        },
+        {
+          "name": "bootloader_code_hash",
+          "ordinal": 7,
+          "type_info": "Bytea"
+        },
+        {
+          "name": "default_aa_code_hash",
+          "ordinal": 8,
+          "type_info": "Bytea"
+        },
+        {
+          "name": "virtual_blocks",
+          "ordinal": 9,
+          "type_info": "Int8"
+        },
+        {
+          "name": "hash",
+          "ordinal": 10,
+          "type_info": "Bytea"
+        },
+        {
+          "name": "consensus",
+          "ordinal": 11,
+          "type_info": "Jsonb"
+        },
+        {
+          "name": "protocol_version!",
+          "ordinal": 12,
+          "type_info": "Int4"
+        },
+        {
+          "name": "fee_account_address?",
+          "ordinal": 13,
+          "type_info": "Bytea"
+        }
+      ],
+      "nullable": [
+        false,
+        null,
+        null,
+        false,
+        false,
+        false,
+        false,
+        true,
+        true,
+        false,
+        false,
+        true,
+        true,
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Int8"
+        ]
+      }
+    },
+    "query": "SELECT miniblocks.number, COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", miniblocks.timestamp, miniblocks.hash as \"root_hash?\", miniblocks.l1_gas_price, miniblocks.l2_fair_gas_price, miniblocks.bootloader_code_hash, miniblocks.default_aa_code_hash, miniblocks.virtual_blocks, miniblocks.hash, miniblocks.consensus, miniblocks.protocol_version as \"protocol_version!\", l1_batches.fee_account_address as \"fee_account_address?\" FROM miniblocks LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number WHERE miniblocks.number = $1"
+  },
   "79cdb4cdd3c47b3654e6240178985fb4b4420e0634f9482a6ef8169e90200b84": {
     "describe": {
       "columns": [
@@ -12321,6 +12327,19 @@
     },
     "query": "SELECT * FROM eth_txs WHERE id = $1"
   },
+  "fcca1961f34082f7186de607b922fd608166c5af98031e4dcc8a056b89696dbe": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Int8",
+          "Jsonb"
+        ]
+      }
+    },
+    "query": "UPDATE miniblocks SET consensus = $2 WHERE number = $1"
+  },
   "ff7ff36b86b0e8d1cd7280aa447baef172cb054ffe7e1d742c59bf09b4f414cb": {
     "describe": {
       "columns": [

core/lib/dal/src/blocks_dal.rs

@@ -10,7 +10,7 @@ use sqlx::Row;
 
 use zksync_types::{
     aggregated_operations::AggregatedActionType,
-    block::{BlockGasCount, L1BatchHeader, MiniblockHeader},
+    block::{BlockGasCount, ConsensusBlockFields, L1BatchHeader, MiniblockHeader},
     commitment::{L1BatchMetadata, L1BatchWithMetadata},
     Address, L1BatchNumber, LogQuery, MiniblockNumber, ProtocolVersionId, H256,
     MAX_GAS_PER_PUBDATA_BYTE, U256,
@@ -466,6 +466,27 @@ impl BlocksDal<'_, '_> {
         Ok(())
     }
 
+    /// Sets consensus-related fields for the specified miniblock.
+    pub async fn set_miniblock_consensus_fields(
+        &mut self,
+        miniblock_number: MiniblockNumber,
+        consensus: &ConsensusBlockFields,
+    ) -> anyhow::Result<()> {
+        let result = sqlx::query!(
+            "UPDATE miniblocks SET consensus = $2 WHERE number = $1",
+            miniblock_number.0 as i64,
+            serde_json::to_value(consensus).unwrap(),
+        )
+        .execute(self.storage.conn())
+        .await?;
+
+        anyhow::ensure!(
+            result.rows_affected() == 1,
+            "Miniblock #{miniblock_number} is not present in Postgres"
+        );
+        Ok(())
+    }
+
     pub async fn update_hashes(
         &mut self,
         number_and_hashes: &[(MiniblockNumber, H256)],

core/lib/dal/src/models/storage_sync.rs

@@ -2,8 +2,7 @@ use std::convert::TryInto;
 
 use zksync_contracts::BaseSystemContractsHashes;
 use zksync_types::api::en::SyncBlock;
-use zksync_types::Transaction;
-use zksync_types::{Address, L1BatchNumber, MiniblockNumber, H256};
+use zksync_types::{Address, L1BatchNumber, MiniblockNumber, Transaction, H256};
 
 #[derive(Debug, Clone, sqlx::FromRow)]
 pub struct StorageSyncBlock {
@@ -22,6 +21,7 @@ pub struct StorageSyncBlock {
     pub protocol_version: i32,
     pub virtual_blocks: i64,
     pub hash: Vec<u8>,
+    pub consensus: Option<serde_json::Value>,
 }
 
 impl StorageSyncBlock {
@@ -30,12 +30,14 @@ impl StorageSyncBlock {
         current_operator_address: Address,
         transactions: Option<Vec<Transaction>>,
     ) -> SyncBlock {
+        let number = self.number;
+
         SyncBlock {
             number: MiniblockNumber(self.number as u32),
             l1_batch_number: L1BatchNumber(self.l1_batch_number as u32),
             last_in_batch: self
                 .last_batch_miniblock
-                .map(|n| n == self.number)
+                .map(|n| n == number)
                 .unwrap_or(false),
             timestamp: self.timestamp as u64,
             root_hash: self.root_hash.as_deref().map(H256::from_slice),
@@ -60,6 +62,7 @@ impl StorageSyncBlock {
             virtual_blocks: Some(self.virtual_blocks as u32),
             hash: Some(H256::from_slice(&self.hash)),
             protocol_version: (self.protocol_version as u16).try_into().unwrap(),
+            consensus: self.consensus.map(|v| serde_json::from_value(v).unwrap()),
         }
     }
 }

core/lib/dal/src/sync_dal.rs

@@ -23,24 +23,23 @@ impl SyncDal<'_, '_> {
         let latency = MethodLatency::new("sync_dal_sync_block");
         let storage_block_details = sqlx::query_as!(
             StorageSyncBlock,
-            r#"
-                SELECT miniblocks.number,
-                    COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as "l1_batch_number!",
-                    (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as "last_batch_miniblock?",
-                    miniblocks.timestamp,
-                    miniblocks.hash as "root_hash?",
-                    miniblocks.l1_gas_price,
-                    miniblocks.l2_fair_gas_price,
-                    miniblocks.bootloader_code_hash,
-                    miniblocks.default_aa_code_hash,
-                    miniblocks.virtual_blocks,
-                    miniblocks.hash,
-                    miniblocks.protocol_version as "protocol_version!",
-                    l1_batches.fee_account_address as "fee_account_address?"
-                FROM miniblocks
-                LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number
-                WHERE miniblocks.number = $1
-            "#,
+            "SELECT miniblocks.number, \
+                COALESCE(miniblocks.l1_batch_number, (SELECT (max(number) + 1) FROM l1_batches)) as \"l1_batch_number!\", \
+                (SELECT max(m2.number) FROM miniblocks m2 WHERE miniblocks.l1_batch_number = m2.l1_batch_number) as \"last_batch_miniblock?\", \
+                miniblocks.timestamp, \
+                miniblocks.hash as \"root_hash?\", \
+                miniblocks.l1_gas_price, \
+                miniblocks.l2_fair_gas_price, \
+                miniblocks.bootloader_code_hash, \
+                miniblocks.default_aa_code_hash, \
+                miniblocks.virtual_blocks, \
+                miniblocks.hash, \
+                miniblocks.consensus, \
+                miniblocks.protocol_version as \"protocol_version!\", \
+                l1_batches.fee_account_address as \"fee_account_address?\" \
+            FROM miniblocks \
+            LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number \
+            WHERE miniblocks.number = $1",
             block_number.0 as i64
         )
         .instrument("sync_dal_sync_block.block")

core/lib/types/Cargo.toml

@@ -10,6 +10,8 @@ keywords = ["blockchain", "zksync"]
 categories = ["cryptography"]
 readme = "README.md"
 
+links = "zksync_types_proto"
+
 [dependencies]
 zksync_system_constants = { path = "../constants" }
 zksync_utils = { path = "../utils" }
@@ -21,10 +23,14 @@ codegen = { git = "https://github.com/matter-labs/solidity_plonk_verifier.git",
 zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.3.3" }
 zk_evm_1_4_0 = { git = "https://github.com/matter-labs/era-zk_evm.git", branch = "v1.4.0", package = "zk_evm" }
 zk_evm = { git = "https://github.com/matter-labs/era-zk_evm.git", tag = "v1.3.3-rc2" }
+zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" }
+zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" }
 
+anyhow = "1.0.75"
 chrono = { version = "0.4", features = ["serde"] }
 num = { version = "0.3.1", features = ["serde"] }
 once_cell = "1.7"
+prost = "0.12.1"
 rlp = "0.5"
 serde = "1.0.90"
 serde_json = "1.0.0"
@@ -47,3 +53,6 @@ ethereum_types_old = { package = "ethereum-types", version = "0.12.0" }
 secp256k1 = { version = "0.27", features = ["recovery"] }
 tokio = { version = "1", features = ["rt", "macros"] }
 serde_with = { version = "1", features = ["hex"] }
+
+[build-dependencies]
+zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "bdf9ed0af965cc7fa32d6c46a35ea065779ede8b" }

core/lib/types/build.rs

@@ -0,0 +1,12 @@
+//! Generates rust code from protobufs.
+fn main() {
+    zksync_protobuf_build::Config {
+        input_root: "src/proto".into(),
+        proto_root: "zksync/types".into(),
+        dependencies: vec!["::zksync_consensus_roles::proto".parse().unwrap()],
+        protobuf_crate: "::zksync_protobuf".parse().unwrap(),
+        is_public: true,
+    }
+    .generate()
+    .expect("generate()");
+}