Merge remote-tracking branch 'origin/main' into daniyar/consensus/sto…

…re-traces

.github/release-please/manifest.json

@@ -1,4 +1,4 @@
 {
-  "core": "24.12.0",
+  "core": "24.13.0",
   "prover": "16.1.0"
 }

Cargo.toml

@@ -143,6 +143,7 @@ opentelemetry = "0.24.0"
 opentelemetry_sdk = "0.24.0"
 opentelemetry-otlp = "0.17.0"
 opentelemetry-semantic-conventions = "0.16.0"
+opentelemetry-appender-tracing = "0.5"
 pin-project-lite = "0.2.13"
 pretty_assertions = "1"
 prost = "0.12.1"

core/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## [24.13.0](https://github.com/matter-labs/zksync-era/compare/core-v24.12.0...core-v24.13.0) (2024-07-31)
+
+
+### Features
+
+* Add recovery tests to zk_supervisor ([#2444](https://github.com/matter-labs/zksync-era/issues/2444)) ([0c0d10a](https://github.com/matter-labs/zksync-era/commit/0c0d10af703d3f8958c49d0ed46d6cda64945fa1))
+* Added a JSON RPC to simulating L1 for consensus attestation ([#2480](https://github.com/matter-labs/zksync-era/issues/2480)) ([c6b3adf](https://github.com/matter-labs/zksync-era/commit/c6b3adf3f29d3a89daa2cfffa1c0e5cb9770eb0d))
+* added dropping all attester certificates when doing hard fork ([#2529](https://github.com/matter-labs/zksync-era/issues/2529)) ([5acd686](https://github.com/matter-labs/zksync-era/commit/5acd68640da6b22897185d76101180bcd838ac67))
+* **configs:** Do not panic if config is only partially filled ([#2545](https://github.com/matter-labs/zksync-era/issues/2545)) ([db13fe3](https://github.com/matter-labs/zksync-era/commit/db13fe3550598c69f59cd66b4bb9618ebea041ca))
+* **eth-sender:** Make eth-sender tests use blob txs + refactor of eth-sender tests ([#2316](https://github.com/matter-labs/zksync-era/issues/2316)) ([c8c8334](https://github.com/matter-labs/zksync-era/commit/c8c83349c10710d75c0030409f926db313c9660d))
+* Introduce more tracing instrumentation ([#2523](https://github.com/matter-labs/zksync-era/issues/2523)) ([79d407a](https://github.com/matter-labs/zksync-era/commit/79d407ac47ac51667196aa2cd028d05b1622130f))
+* Remove unused VKs, add docs for BWG ([#2468](https://github.com/matter-labs/zksync-era/issues/2468)) ([2fa6bf0](https://github.com/matter-labs/zksync-era/commit/2fa6bf0ffa5d7a5ff62d595a0efeff9dcd9e5a1a))
+* Revisit base config values ([#2532](https://github.com/matter-labs/zksync-era/issues/2532)) ([3fac8ac](https://github.com/matter-labs/zksync-era/commit/3fac8ac62cc9ac14845f32240af9241386f4034d))
+* Server 10k gwei limit on gas price and 1M limit on pubdata price ([#2460](https://github.com/matter-labs/zksync-era/issues/2460)) ([be238cc](https://github.com/matter-labs/zksync-era/commit/be238ccb35e4581de22156f76903211bd85526b7))
+* **vlog:** Implement otlp guard with force flush on drop ([#2536](https://github.com/matter-labs/zksync-era/issues/2536)) ([c9f76e5](https://github.com/matter-labs/zksync-era/commit/c9f76e571e5570d2c4194feee03bb260b24c378f))
+* **vlog:** New vlog interface + opentelemtry improvements ([#2472](https://github.com/matter-labs/zksync-era/issues/2472)) ([c0815cd](https://github.com/matter-labs/zksync-era/commit/c0815cdaf878afcd9c41dddd9fe56bcf8d910633))
+* **zk_toolbox:** add test upgrade subcommand to zk_toolbox ([#2515](https://github.com/matter-labs/zksync-era/issues/2515)) ([1a12f5f](https://github.com/matter-labs/zksync-era/commit/1a12f5f908add42c090170a2f4fb26b731d6971b))
+* **zk_toolbox:** use configs from the main repo ([#2470](https://github.com/matter-labs/zksync-era/issues/2470)) ([4222d13](https://github.com/matter-labs/zksync-era/commit/4222d135b62eb4de103c4aebb35e9c302d94ad63))
+
+
+### Bug Fixes
+
+* **contract verifier:** Fix config values  ([#2510](https://github.com/matter-labs/zksync-era/issues/2510)) ([3729468](https://github.com/matter-labs/zksync-era/commit/3729468436114642e62ce8a531533921015455a7))
+* fixed panic propagation ([#2525](https://github.com/matter-labs/zksync-era/issues/2525)) ([e0fc58b](https://github.com/matter-labs/zksync-era/commit/e0fc58b536debf804920887e3eb4bc050a9fe9d6))
+* **proof_data_handler:** Unlock jobs on transient errors ([#2486](https://github.com/matter-labs/zksync-era/issues/2486)) ([7c336b1](https://github.com/matter-labs/zksync-era/commit/7c336b1e180b9d5ba1ba74169c61ce27a251e2fc))
+* **prover:** Parallelize circuit metadata uploading for BWG ([#2520](https://github.com/matter-labs/zksync-era/issues/2520)) ([f49720f](https://github.com/matter-labs/zksync-era/commit/f49720fbafdab8f102d908b2be3fa869482a92fa))
+* VM performance diff: don't show 0 as N/A ([#2276](https://github.com/matter-labs/zksync-era/issues/2276)) ([2fa2249](https://github.com/matter-labs/zksync-era/commit/2fa2249dca15b1968fceec11e485850395f03c9d))
+
 ## [24.12.0](https://github.com/matter-labs/zksync-era/compare/core-v24.11.0...core-v24.12.0) (2024-07-25)
 
 

core/bin/external_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "zksync_external_node"
 description = "Non-validator ZKsync node"
-version = "24.12.0" # x-release-please-version
+version = "24.13.0" # x-release-please-version
 edition.workspace = true
 authors.workspace = true
 homepage.workspace = true

core/lib/config/src/configs/observability.rs

@@ -19,6 +19,12 @@ pub struct ObservabilityConfig {
 pub struct OpentelemetryConfig {
     /// Enables export of span data of specified level (and above) using opentelemetry exporters.
     pub level: String,
-    /// Opentelemetry HTTP collector endpoint.
+    /// Opentelemetry HTTP traces collector endpoint.
     pub endpoint: String,
+    /// Opentelemetry HTTP logs collector endpoing.
+    /// This is optional, since right now the primary way to collect logs is via stdout.
+    ///
+    /// Important: sending logs via OTLP has only been tested locally, and the performance may be
+    /// suboptimal in production environments.
+    pub logs_endpoint: Option<String>,
 }

core/lib/config/src/observability_ext.rs

@@ -46,7 +46,13 @@ impl TryFrom<ObservabilityConfig> for Option<zksync_vlog::OpenTelemetry> {
     fn try_from(config: ObservabilityConfig) -> Result<Self, Self::Error> {
         Ok(config
             .opentelemetry
-            .map(|config| zksync_vlog::OpenTelemetry::new(&config.level, config.endpoint))
+            .map(|config| {
+                zksync_vlog::OpenTelemetry::new(
+                    &config.level,
+                    Some(config.endpoint),
+                    config.logs_endpoint,
+                )
+            })
             .transpose()?)
     }
 }

core/lib/config/src/testonly.rs

@@ -681,6 +681,7 @@ impl Distribution<configs::OpentelemetryConfig> for EncodeDist {
         configs::OpentelemetryConfig {
             level: self.sample(rng),
             endpoint: self.sample(rng),
+            logs_endpoint: self.sample(rng),
         }
     }
 }

core/lib/dal/build.rs

@@ -3,7 +3,7 @@ fn main() {
     zksync_protobuf_build::Config {
         input_root: "src/consensus/proto".into(),
         proto_root: "zksync/dal".into(),
-        dependencies: vec![],
+        dependencies: vec!["::zksync_consensus_roles::proto".parse().unwrap()],
         protobuf_crate: "::zksync_protobuf".parse().unwrap(),
         is_public: true,
     }

core/lib/dal/src/consensus/mod.rs

@@ -1,11 +1,13 @@
 pub mod proto;
 
+#[cfg(test)]
+mod testonly;
 #[cfg(test)]
 mod tests;
 
 use anyhow::{anyhow, Context as _};
-use zksync_consensus_roles::validator;
-use zksync_protobuf::{required, ProtoFmt, ProtoRepr};
+use zksync_consensus_roles::{attester, validator};
+use zksync_protobuf::{read_required, required, ProtoFmt, ProtoRepr};
 use zksync_types::{
     abi, ethabi,
     fee::Fee,
@@ -20,6 +22,34 @@ use zksync_utils::{h256_to_u256, u256_to_h256};
 
 use crate::models::{parse_h160, parse_h256};
 
+/// Global attestation status served by
+/// `attestationStatus` RPC.
+#[derive(Debug, PartialEq, Clone)]
+pub struct AttestationStatus {
+    pub genesis: validator::GenesisHash,
+    pub next_batch_to_attest: attester::BatchNumber,
+}
+
+impl ProtoFmt for AttestationStatus {
+    type Proto = proto::AttestationStatus;
+
+    fn read(r: &Self::Proto) -> anyhow::Result<Self> {
+        Ok(Self {
+            genesis: read_required(&r.genesis).context("genesis")?,
+            next_batch_to_attest: attester::BatchNumber(
+                *required(&r.next_batch_to_attest).context("next_batch_to_attest")?,
+            ),
+        })
+    }
+
+    fn build(&self) -> Self::Proto {
+        Self::Proto {
+            genesis: Some(self.genesis.build()),
+            next_batch_to_attest: Some(self.next_batch_to_attest.0),
+        }
+    }
+}
+
 /// L2 block (= miniblock) payload.
 #[derive(Debug, PartialEq)]
 pub struct Payload {

core/lib/dal/src/consensus/proto/mod.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package zksync.dal;
 
+import "zksync/roles/validator.proto";
+
 message Payload {
   // zksync-era ProtocolVersionId
   optional uint32 protocol_version = 9; // required; u16
@@ -114,3 +116,8 @@ message PaymasterParams {
   optional bytes paymaster_address = 1; // required; H160
   optional bytes paymaster_input = 2; // required
 }
+
+message AttestationStatus {
+  optional roles.validator.GenesisHash genesis = 1; // required
+  optional uint64 next_batch_to_attest = 2; // required
+}

core/lib/dal/src/consensus/testonly.rs

@@ -0,0 +1,15 @@
+use rand::{
+    distributions::{Distribution, Standard},
+    Rng,
+};
+
+use super::AttestationStatus;
+
+impl Distribution<AttestationStatus> for Standard {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> AttestationStatus {
+        AttestationStatus {
+            genesis: rng.gen(),
+            next_batch_to_attest: rng.gen(),
+        }
+    }
+}

core/lib/dal/src/consensus/tests.rs

@@ -4,15 +4,15 @@ use rand::Rng;
 use zksync_concurrency::ctx;
 use zksync_protobuf::{
     repr::{decode, encode},
-    testonly::test_encode,
+    testonly::{test_encode, test_encode_random},
     ProtoRepr,
 };
 use zksync_test_account::Account;
 use zksync_types::{
     web3::Bytes, Execute, ExecuteTransactionCommon, L1BatchNumber, ProtocolVersionId, Transaction,
 };
 
-use super::{proto, Payload};
+use super::{proto, AttestationStatus, Payload};
 use crate::tests::mock_protocol_upgrade_transaction;
 
 fn execute(rng: &mut impl Rng) -> Execute {
@@ -59,6 +59,7 @@ fn payload(rng: &mut impl Rng, protocol_version: ProtocolVersionId) -> Payload {
 fn test_encoding() {
     let ctx = &ctx::test_root(&ctx::RealClock);
     let rng = &mut ctx.rng();
+    test_encode_random::<AttestationStatus>(rng);
     encode_decode::<proto::TransactionV25, ComparableTransaction>(l1_transaction(rng));
     encode_decode::<proto::TransactionV25, ComparableTransaction>(l2_transaction(rng));
     encode_decode::<proto::Transaction, ComparableTransaction>(l1_transaction(rng));

core/lib/dal/src/consensus_dal.rs

@@ -9,7 +9,7 @@ use zksync_db_connection::{
 };
 use zksync_types::L2BlockNumber;
 
-pub use crate::consensus::Payload;
+pub use crate::consensus::{AttestationStatus, Payload};
 use crate::{Core, CoreDal};
 
 /// Storage access methods for `zksync_core::consensus` module.
@@ -466,38 +466,82 @@ impl ConsensusDal<'_, '_> {
         )))
     }
 
-    /// Next batch that the attesters should vote for.
+    /// Number of L1 batch that the L2 block belongs to.
+    /// None if the L2 block doesn't exist.
+    async fn batch_of_block(
+        &mut self,
+        block: validator::BlockNumber,
+    ) -> anyhow::Result<Option<attester::BatchNumber>> {
+        let Some(row) = sqlx::query!(
+            r#"
+            SELECT
+                COALESCE(
+                    miniblocks.l1_batch_number,
+                    (
+                        SELECT
+                            (MAX(number) + 1)
+                        FROM
+                            l1_batches
+                    ),
+                    (
+                        SELECT
+                            MAX(l1_batch_number) + 1
+                        FROM
+                            snapshot_recovery
+                    )
+                ) AS "l1_batch_number!"
+            FROM
+                miniblocks
+            WHERE
+                number = $1
+            "#,
+            i64::try_from(block.0).context("overflow")?,
+        )
+        .instrument("batch_of_block")
+        .report_latency()
+        .fetch_optional(self.storage)
+        .await?
+        else {
+            return Ok(None);
+        };
+        Ok(Some(attester::BatchNumber(
+            row.l1_batch_number.try_into().context("overflow")?,
+        )))
+    }
+
+    /// Global attestation status.
+    /// Includes the next batch that the attesters should vote for.
+    /// None iff the consensus genesis is missing (i.e. consensus wasn't enabled) or
+    /// L2 block with number `genesis.first_block` doesn't exist yet.
+    ///
     /// This is a main node only query.
     /// ENs should call the attestation_status RPC of the main node.
-    pub async fn next_batch_to_attest(&mut self) -> anyhow::Result<attester::BatchNumber> {
-        // First batch that we don't have a certificate for.
-        if let Some(last) = self
-            .get_last_batch_certificate_number()
-            .await
-            .context("get_last_batch_certificate_number()")?
-        {
-            return Ok(last + 1);
-        }
-        // Otherwise start with the last sealed L1 batch.
-        // We don't want to backfill certificates for old batches.
-        // Note that there is a race condition in case the next
-        // batch is sealed before the certificate for the current
-        // last sealed batch is stored. This is only relevant
-        // for the first certificate though and anyway this is
-        // a test setup, so we are OK with that race condition.
-        if let Some(sealed) = self
-            .storage
-            .blocks_dal()
-            .get_sealed_l1_batch_number()
-            .await
-            .context("get_sealed_l1_batch_number()")?
-        {
-            return Ok(attester::BatchNumber(sealed.0.into()));
+    pub async fn attestation_status(&mut self) -> anyhow::Result<Option<AttestationStatus>> {
+        let Some(genesis) = self.genesis().await.context("genesis()")? else {
+            return Ok(None);
+        };
+        let Some(next_batch_to_attest) = async {
+            // First batch that we don't have a certificate for.
+            if let Some(last) = self
+                .get_last_batch_certificate_number()
+                .await
+                .context("get_last_batch_certificate_number()")?
+            {
+                return Ok(Some(last + 1));
+            }
+            // Otherwise start with the batch containing the first block of the fork.
+            self.batch_of_block(genesis.first_block)
+                .await
+                .context("batch_of_block()")
         }
-        // Otherwise start with 0.
-        // Note that main node doesn't start from snapshot
-        // and doesn't have prunning enabled.
-        Ok(attester::BatchNumber(0))
+        .await?
+        else {
+            return Ok(None);
+        };
+        Ok(Some(AttestationStatus {
+            genesis: genesis.hash(),
+            next_batch_to_attest,
+        }))
     }
 }