From 67653dadd9c8c09ecc7bd6765be45df4907419ce Mon Sep 17 00:00:00 2001 From: Marek Date: Fri, 8 Dec 2023 18:37:04 +0100 Subject: [PATCH 1/7] Allow opening the database read-only mode --- zebra-scan/src/init.rs | 2 +- zebra-scan/src/storage.rs | 4 ++-- zebra-scan/src/storage/db.rs | 6 ++++-- zebra-scan/src/tests.rs | 2 +- zebra-state/src/service/finalized_state.rs | 3 +++ zebra-state/src/service/finalized_state/disk_db.rs | 7 ++++++- zebra-state/src/service/finalized_state/zebra_db.rs | 2 ++ .../finalized_state/zebra_db/block/tests/vectors.rs | 1 + zebra-state/src/service/read/tests/vectors.rs | 1 + zebra-state/src/tests/setup.rs | 1 + 10 files changed, 22 insertions(+), 7 deletions(-) diff --git a/zebra-scan/src/init.rs b/zebra-scan/src/init.rs index a94676ead41..03cea9a90f1 100644 --- a/zebra-scan/src/init.rs +++ b/zebra-scan/src/init.rs @@ -33,7 +33,7 @@ pub async fn init( state: scan::State, chain_tip_change: ChainTipChange, ) -> Result<(), Report> { - let storage = tokio::task::spawn_blocking(move || Storage::new(&config, network)) + let storage = tokio::task::spawn_blocking(move || Storage::new(&config, network, false)) .wait_for_panics() .await; diff --git a/zebra-scan/src/storage.rs b/zebra-scan/src/storage.rs index c4324ceda45..3fabc268098 100644 --- a/zebra-scan/src/storage.rs +++ b/zebra-scan/src/storage.rs @@ -55,8 +55,8 @@ impl Storage { /// /// This method can block while creating or reading database files, so it must be inside /// spawn_blocking() in async code. - pub fn new(config: &Config, network: Network) -> Self { - let mut storage = Self::new_db(config, network); + pub fn new(config: &Config, network: Network, read_only: bool) -> Self { + let mut storage = Self::new_db(config, network, read_only); for (sapling_key, birthday) in config.sapling_keys_to_scan.iter() { storage.add_sapling_key(sapling_key, Some(zebra_chain::block::Height(*birthday))); diff --git a/zebra-scan/src/storage/db.rs b/zebra-scan/src/storage/db.rs index 00262b01437..f6782b21ab5 100644 --- a/zebra-scan/src/storage/db.rs +++ b/zebra-scan/src/storage/db.rs @@ -46,11 +46,11 @@ impl Storage { /// If there is no existing database, creates a new database on disk. /// /// New keys in `config` are not inserted into the database. - pub(crate) fn new_db(config: &Config, network: Network) -> Self { + pub(crate) fn new_db(config: &Config, network: Network, read_only: bool) -> Self { Self::new_with_debug( config, network, // TODO: make format upgrades work with any database, then change this to `false` - true, + true, read_only, ) } @@ -64,6 +64,7 @@ impl Storage { config: &Config, network: Network, debug_skip_format_upgrades: bool, + read_only: bool, ) -> Self { let db = ScannerDb::new( config.db_config(), @@ -74,6 +75,7 @@ impl Storage { SCANNER_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + read_only, ); let new_storage = Self { db }; diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index 99b8c87d907..0763be033dd 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -186,7 +186,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> { zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); // Create a database - let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet); + let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet, false); // Insert the generated key to the database s.add_sapling_key(&key_to_be_stored, None); diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 2e923fe7330..d507fdd57d8 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -143,6 +143,7 @@ impl FinalizedState { false, #[cfg(feature = "elasticsearch")] elastic_db, + false, ) } @@ -155,6 +156,7 @@ impl FinalizedState { network: Network, debug_skip_format_upgrades: bool, #[cfg(feature = "elasticsearch")] elastic_db: Option, + read_only: bool, ) -> Self { let db = ZebraDb::new( config, @@ -165,6 +167,7 @@ impl FinalizedState { STATE_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + read_only, ); #[cfg(feature = "elasticsearch")] diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index ab20a62faa3..013ca606d38 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -622,6 +622,7 @@ impl DiskDb { format_version_in_code: &Version, network: Network, column_families_in_code: impl IntoIterator, + read_only: bool, ) -> DiskDb { let db_kind = db_kind.as_ref(); let path = config.db_path(db_kind, format_version_in_code.major, network); @@ -644,7 +645,11 @@ impl DiskDb { .unique() .map(|cf_name| rocksdb::ColumnFamilyDescriptor::new(cf_name, db_options.clone())); - let db_result = DB::open_cf_descriptors(&db_options, &path, column_families); + let db_result = if read_only { + DB::open_cf_descriptors_read_only(&db_options, &path, column_families, false) + } else { + DB::open_cf_descriptors(&db_options, &path, column_families) + }; match db_result { Ok(db) => { diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index 6b7c6823dcd..c2f9e724ce4 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -96,6 +96,7 @@ impl ZebraDb { network: Network, debug_skip_format_upgrades: bool, column_families_in_code: impl IntoIterator, + read_only: bool, ) -> ZebraDb { let disk_version = database_format_version_on_disk( config, @@ -129,6 +130,7 @@ impl ZebraDb { format_version_in_code, network, column_families_in_code, + read_only, ), }; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 4a3385ebe8a..f0ce0d0f414 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -89,6 +89,7 @@ fn test_block_db_round_trip_with( STATE_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + false, ); // Check that each block round-trips to the database diff --git a/zebra-state/src/service/read/tests/vectors.rs b/zebra-state/src/service/read/tests/vectors.rs index 38d0887aaec..f1f39b6ccc4 100644 --- a/zebra-state/src/service/read/tests/vectors.rs +++ b/zebra-state/src/service/read/tests/vectors.rs @@ -377,5 +377,6 @@ fn new_ephemeral_db() -> ZebraDb { STATE_COLUMN_FAMILIES_IN_CODE .iter() .map(ToString::to_string), + false, ) } diff --git a/zebra-state/src/tests/setup.rs b/zebra-state/src/tests/setup.rs index 1432e72f368..c5a900c0507 100644 --- a/zebra-state/src/tests/setup.rs +++ b/zebra-state/src/tests/setup.rs @@ -99,6 +99,7 @@ pub(crate) fn new_state_with_mainnet_genesis( true, #[cfg(feature = "elasticsearch")] None, + false, ); let non_finalized_state = NonFinalizedState::new(network); From 81ad7ed992ff03ed93dce5d7ae4265580e2d0b34 Mon Sep 17 00:00:00 2001 From: Marek Date: Mon, 11 Dec 2023 20:48:09 +0100 Subject: [PATCH 2/7] Update zebra-scan/src/storage/db.rs Co-authored-by: teor --- zebra-scan/src/storage/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scan/src/storage/db.rs b/zebra-scan/src/storage/db.rs index f6782b21ab5..4e6696d8c12 100644 --- a/zebra-scan/src/storage/db.rs +++ b/zebra-scan/src/storage/db.rs @@ -49,7 +49,7 @@ impl Storage { pub(crate) fn new_db(config: &Config, network: Network, read_only: bool) -> Self { Self::new_with_debug( config, network, - // TODO: make format upgrades work with any database, then change this to `false` + // TODO: make format upgrades work with any database, then change debug_skip_format_upgrades to `false` true, read_only, ) } From 06904cbbc4c283c7d771099e8adba47806414228 Mon Sep 17 00:00:00 2001 From: Marek Date: Mon, 11 Dec 2023 21:31:23 +0100 Subject: [PATCH 3/7] Refactor skipping database upgrades --- zebra-state/src/service/finalized_state/zebra_db.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index c2f9e724ce4..810feff878e 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -109,16 +109,20 @@ impl ZebraDb { // Log any format changes before opening the database, in case opening fails. let format_change = DbFormatChange::open_database(format_version_in_code, disk_version); - // Always do format upgrades in production, but allow them to be skipped by the scanner - // (because it doesn't support them yet). + // Format upgrades try to write to the database, so we always skip them if `read_only` is + // `true`. + // + // We allow skipping the upgrades by the scanner because it doesn't support them yet and we + // also allow skipping them when we are running tests. // // TODO: Make scanner support format upgrades, then remove `shielded-scan` here. - let can_skip_format_upgrades = cfg!(test) || cfg!(feature = "shielded-scan"); + let debug_skip_format_upgrades = read_only + || ((cfg!(test) || cfg!(feature = "shielded-scan")) && debug_skip_format_upgrades); // Open the database and do initial checks. let mut db = ZebraDb { config: Arc::new(config.clone()), - debug_skip_format_upgrades: can_skip_format_upgrades && debug_skip_format_upgrades, + debug_skip_format_upgrades, format_change_handle: None, // After the database directory is created, a newly created database temporarily // changes to the default database version. Then we set the correct version in the From a72029473fa05ee4dfa89d2323521ec9c492fc84 Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 12 Dec 2023 14:15:54 +0100 Subject: [PATCH 4/7] Fix errors caused by the merge --- zebra-scan/src/storage/db/tests/snapshot.rs | 2 +- zebra-scan/src/tests/vectors.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-scan/src/storage/db/tests/snapshot.rs b/zebra-scan/src/storage/db/tests/snapshot.rs index 0eaef193045..48879cc9453 100644 --- a/zebra-scan/src/storage/db/tests/snapshot.rs +++ b/zebra-scan/src/storage/db/tests/snapshot.rs @@ -57,7 +57,7 @@ fn test_raw_rocksdb_column_families_with_network(network: Network) { let mut net_suffix = network.to_string(); net_suffix.make_ascii_lowercase(); - let mut storage = Storage::new(&Config::ephemeral(), network); + let mut storage = Storage::new(&Config::ephemeral(), network, false); // Snapshot the column family names let mut cf_names = storage.db.list_cf().expect("empty database is valid"); diff --git a/zebra-scan/src/tests/vectors.rs b/zebra-scan/src/tests/vectors.rs index bcdca24c6e7..176608be950 100644 --- a/zebra-scan/src/tests/vectors.rs +++ b/zebra-scan/src/tests/vectors.rs @@ -157,7 +157,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> { zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); // Create a database - let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet); + let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet, false); // Insert the generated key to the database s.add_sapling_key(&key_to_be_stored, None); From f7cb32a9002d3f2e9c5ea86ccc5bb726ab22bd99 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 13 Dec 2023 08:45:34 +1000 Subject: [PATCH 5/7] Add new argument in new_test_storage() --- zebra-scan/src/storage/db/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scan/src/storage/db/tests.rs b/zebra-scan/src/storage/db/tests.rs index 6b6365be981..b813a8b91bc 100644 --- a/zebra-scan/src/storage/db/tests.rs +++ b/zebra-scan/src/storage/db/tests.rs @@ -20,7 +20,7 @@ mod snapshot; /// Returns an empty `Storage` suitable for testing. pub fn new_test_storage(network: Network) -> Storage { - Storage::new(&Config::ephemeral(), network) + Storage::new(&Config::ephemeral(), network, false) } /// Add fake keys to `storage` for testing purposes. From ed19740fdb86b739852d9403b1b5b3d77359477a Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 13 Dec 2023 08:47:12 +1000 Subject: [PATCH 6/7] Simplify test storage code --- zebra-scan/src/tests/vectors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scan/src/tests/vectors.rs b/zebra-scan/src/tests/vectors.rs index 176608be950..52250826a1e 100644 --- a/zebra-scan/src/tests/vectors.rs +++ b/zebra-scan/src/tests/vectors.rs @@ -157,7 +157,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> { zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); // Create a database - let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet, false); + let mut s = crate::storage::tests::new_test_storage(Network::Mainnet); // Insert the generated key to the database s.add_sapling_key(&key_to_be_stored, None); From 649cbfde471aecaafcec1e689d2115a6ab5d239d Mon Sep 17 00:00:00 2001 From: Marek Date: Wed, 13 Dec 2023 14:49:57 +0100 Subject: [PATCH 7/7] Fix importing the `new_test_storate` fn --- zebra-scan/src/tests/vectors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-scan/src/tests/vectors.rs b/zebra-scan/src/tests/vectors.rs index 52250826a1e..9fa6734b020 100644 --- a/zebra-scan/src/tests/vectors.rs +++ b/zebra-scan/src/tests/vectors.rs @@ -22,8 +22,8 @@ use zebra_chain::{ use zebra_state::{SaplingScannedResult, TransactionIndex}; use crate::{ - config::Config, scan::{block_to_compact, scan_block}, + storage::db::tests::new_test_storage, tests::{fake_block, ZECPAGES_SAPLING_VIEWING_KEY}, }; @@ -157,7 +157,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> { zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); // Create a database - let mut s = crate::storage::tests::new_test_storage(Network::Mainnet); + let mut s = new_test_storage(Network::Mainnet); // Insert the generated key to the database s.add_sapling_key(&key_to_be_stored, None);