fix(tests): add submitblock test to CI, and avoid copying the cached state directory in other tests

.github/workflows/continous-integration-docker.patch.yml

@@ -90,6 +90,12 @@ jobs:
     steps:
       - run: 'echo "No build required"'
 
+  submit-block-test:
+    name: submit block / Run submit-block test
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
   lightwalletd-full-sync:
     name: lightwalletd tip / Run lwd-full-sync test
     runs-on: ubuntu-latest

.github/workflows/continous-integration-docker.yml

@@ -575,3 +575,28 @@ jobs:
       root_state_path: '/var/cache'
       zebra_state_dir: 'zebrad-cache'
       lwd_state_dir: 'lwd-cache'
+
+  # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state
+  #
+  # Runs:
+  # - after every PR is merged to `main`
+  # - on every PR update
+  #
+  # If the state version has changed, waits for the new cached states to be created.
+  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
+  submit-block-test:
+    name: submit block
+    needs: test-full-sync
+    uses: ./.github/workflows/deploy-gcp-tests.yml
+    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }}
+    with:
+      app_name: zebrad
+      test_id: submit-block
+      test_description: Test submitting blocks via Zebra's rpc server
+      test_variables: '-e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
+      needs_zebra_state: true
+      needs_lwd_state: false
+      saves_to_disk: false
+      disk_suffix: tip
+      root_state_path: '/var/cache'
+      zebra_state_dir: 'zebrad-cache'

docker/entrypoint.sh

@@ -78,6 +78,10 @@ case "$1" in
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
             cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
+        elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then
+            # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port.
+            ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
+            cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored submit_block
 
         # These command-lines are provided by the caller.
         #

zebrad/tests/acceptance.rs

@@ -146,20 +146,19 @@ mod common;
 
 use common::{
     check::{is_zebrad_version, EphemeralCheck, EphemeralConfig},
+    config::random_known_rpc_port_config,
     config::{
         config_file_full_path, configs_dir, default_test_config, persistent_test_config, testdir,
     },
     launch::{spawn_zebrad_for_rpc, ZebradTestDirExt, BETWEEN_NODES_DELAY, LAUNCH_DELAY},
-    lightwalletd::{
-        can_spawn_lightwalletd_for_rpc, random_known_rpc_port_config, spawn_lightwalletd_for_rpc,
-        LightwalletdTestType::{self, *},
-    },
+    lightwalletd::{can_spawn_lightwalletd_for_rpc, spawn_lightwalletd_for_rpc},
     sync::{
         create_cached_database_height, sync_until, MempoolBehavior, LARGE_CHECKPOINT_TEST_HEIGHT,
         LARGE_CHECKPOINT_TIMEOUT, MEDIUM_CHECKPOINT_TEST_HEIGHT, STOP_AT_HEIGHT_REGEX,
         STOP_ON_LOAD_TIMEOUT, SYNC_FINISHED_REGEX, TINY_CHECKPOINT_TEST_HEIGHT,
         TINY_CHECKPOINT_TIMEOUT,
     },
+    test_type::TestType::{self, *},
 };
 
 /// The maximum amount of time that we allow the creation of a future to block the `tokio` executor.
@@ -1571,7 +1570,7 @@ async fn lightwalletd_test_suite() -> Result<()> {
 /// If the `test_type` requires `--features=lightwalletd-grpc-tests`,
 /// but Zebra was not compiled with that feature.
 #[tracing::instrument]
-fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> {
+fn lightwalletd_integration_test(test_type: TestType) -> Result<()> {
     let _init_guard = zebra_test::init();
 
     // We run these sync tests with a network connection, for better test coverage.
@@ -2015,7 +2014,7 @@ async fn fully_synced_rpc_test() -> Result<()> {
     let _init_guard = zebra_test::init();
 
     // We're only using cached Zebra state here, so this test type is the most similar
-    let test_type = LightwalletdTestType::UpdateCachedState;
+    let test_type = TestType::UpdateCachedState;
     let network = Network::Mainnet;
 
     let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) =

zebrad/tests/common/cached_state.rs

@@ -7,11 +7,17 @@
 
 use std::path::{Path, PathBuf};
 
+use std::time::Duration;
+
+use reqwest::Client;
+
 use color_eyre::eyre::{eyre, Result};
 use tempfile::TempDir;
 use tokio::fs;
 use tower::{util::BoxService, Service};
 
+use zebra_chain::block::Block;
+use zebra_chain::serialization::ZcashDeserializeInto;
 use zebra_chain::{
     block::{self, Height},
     chain_tip::ChainTip,
@@ -21,6 +27,14 @@ use zebra_state::{ChainTipChange, LatestChainTip};
 
 use crate::common::config::testdir;
 
+use zebra_state::MAX_BLOCK_REORG_HEIGHT;
+
+use crate::common::{
+    launch::spawn_zebrad_for_rpc,
+    sync::{check_sync_logs_until, MempoolBehavior, SYNC_FINISHED_REGEX},
+    test_type::TestType,
+};
+
 /// Path to a directory containing a cached Zebra state.
 pub const ZEBRA_CACHED_STATE_DIR: &str = "ZEBRA_CACHED_STATE_DIR";
 
@@ -144,3 +158,152 @@ async fn copy_directory(
 
     Ok(sub_directories)
 }
+
+/// Accepts a network, test_type, test_name, and num_blocks (how many blocks past the finalized tip to try getting)
+///
+/// Syncs zebra until the tip, gets some blocks near the tip, via getblock rpc calls,
+/// shuts down zebra, and gets the finalized tip height of the updated cached state.
+///
+/// Returns retrieved and deserialized blocks that are above the finalized tip height of the cached state.
+///
+/// ## Panics
+///
+/// If the provided `test_type` doesn't need an rpc server and cached state, or if `max_num_blocks` is 0
+pub async fn get_future_blocks(
+    network: Network,
+    test_type: TestType,
+    test_name: &str,
+    max_num_blocks: u32,
+) -> Result<Vec<Block>> {
+    let blocks: Vec<Block> = get_raw_future_blocks(network, test_type, test_name, max_num_blocks)
+        .await?
+        .into_iter()
+        .map(hex::decode)
+        .map(|block_bytes| {
+            block_bytes
+                .expect("getblock rpc calls in get_raw_future_blocks should return valid hexdata")
+                .zcash_deserialize_into()
+                .expect("decoded hex data from getblock rpc calls should deserialize into blocks")
+        })
+        .collect();
+
+    Ok(blocks)
+}
+
+/// Accepts a network, test_type, test_name, and num_blocks (how many blocks past the finalized tip to try getting)
+///
+/// Syncs zebra until the tip, gets some blocks near the tip, via getblock rpc calls,
+/// shuts down zebra, and gets the finalized tip height of the updated cached state.
+///
+/// Returns hexdata of retrieved blocks that are above the finalized tip height of the cached state.
+///
+/// ## Panics
+///
+/// If the provided `test_type` doesn't need an rpc server and cached state, or if `max_num_blocks` is 0
+pub async fn get_raw_future_blocks(
+    network: Network,
+    test_type: TestType,
+    test_name: &str,
+    max_num_blocks: u32,
+) -> Result<Vec<String>> {
+    assert!(max_num_blocks > 0);
+
+    let max_num_blocks = max_num_blocks.min(MAX_BLOCK_REORG_HEIGHT);
+    let mut raw_blocks = Vec::with_capacity(max_num_blocks as usize);
+
+    assert!(
+        test_type.needs_zebra_cached_state() && test_type.needs_zebra_rpc_server(),
+        "get_raw_future_blocks needs zebra cached state and rpc server"
+    );
+
+    let should_sync = true;
+    let (zebrad, zebra_rpc_address) =
+        spawn_zebrad_for_rpc(network, test_name, test_type, should_sync)?
+            .ok_or_else(|| eyre!("get_raw_future_blocks requires a cached state"))?;
+    let rpc_address = zebra_rpc_address.expect("test type must have RPC port");
+
+    let mut zebrad = check_sync_logs_until(
+        zebrad,
+        network,
+        SYNC_FINISHED_REGEX,
+        MempoolBehavior::ShouldAutomaticallyActivate,
+        true,
+    )?;
+
+    // Create an http client
+    let client = Client::new();
+
+    let send_rpc_request = |method, params| {
+        client
+            .post(format!("http://{}", &rpc_address))
+            .body(format!(
+                r#"{{"jsonrpc": "2.0", "method": "{method}", "params": {params}, "id":123 }}"#
+            ))
+            .header("Content-Type", "application/json")
+            .send()
+    };
+
+    let blockchain_info: serde_json::Value = serde_json::from_str(
+        &send_rpc_request("getblockchaininfo", "[]".to_string())
+            .await?
+            .text()
+            .await?,
+    )?;
+
+    let tip_height: u32 = blockchain_info["result"]["blocks"]
+        .as_u64()
+        .expect("unexpected block height: doesn't fit in u64")
+        .try_into()
+        .expect("unexpected block height: doesn't fit in u32");
+
+    let estimated_finalized_tip_height = tip_height - MAX_BLOCK_REORG_HEIGHT;
+
+    tracing::info!(
+        ?tip_height,
+        ?estimated_finalized_tip_height,
+        "got tip height from blockchaininfo",
+    );
+
+    for block_height in (0..max_num_blocks).map(|idx| idx + estimated_finalized_tip_height) {
+        let raw_block: serde_json::Value = serde_json::from_str(
+            &send_rpc_request("getblock", format!(r#"["{block_height}", 0]"#))
+                .await?
+                .text()
+                .await?,
+        )?;
+
+        raw_blocks.push((
+            block_height,
+            raw_block["result"]
+                .as_str()
+                .expect("unexpected getblock result: not a string")
+                .to_string(),
+        ));
+    }
+
+    zebrad.kill(true)?;
+
+    // Sleep for a few seconds to make sure zebrad releases lock on cached state directory
+    std::thread::sleep(Duration::from_secs(3));
+
+    let zebrad_state_path = test_type
+        .zebrad_state_path(test_name)
+        .expect("already checked that there is a cached state path");
+
+    let Height(finalized_tip_height) =
+        load_tip_height_from_state_directory(network, zebrad_state_path.as_ref()).await?;
+
+    tracing::info!(
+        ?finalized_tip_height,
+        non_finalized_tip_height = ?tip_height,
+        ?estimated_finalized_tip_height,
+        "got finalized tip height from state directory"
+    );
+
+    let raw_future_blocks = raw_blocks
+        .into_iter()
+        .filter_map(|(height, raw_block)| height.gt(&finalized_tip_height).then_some(raw_block))
+        .collect();
+
+    Ok(raw_future_blocks)
+}

zebrad/tests/common/config.rs

@@ -7,13 +7,15 @@
 
 use std::{
     env,
+    net::SocketAddr,
     path::{Path, PathBuf},
     time::Duration,
 };
 
 use color_eyre::eyre::Result;
 use tempfile::TempDir;
 
+use zebra_test::net::random_known_port;
 use zebrad::{
     components::{mempool, sync, tracing},
     config::ZebradConfig,
@@ -95,3 +97,27 @@ pub fn config_file_full_path(config_file: PathBuf) -> PathBuf {
     let path = configs_dir().join(config_file);
     Path::new(&path).into()
 }
+
+/// Returns a `zebrad` config with a random known RPC port.
+///
+/// Set `parallel_cpu_threads` to true to auto-configure based on the number of CPU cores.
+pub fn random_known_rpc_port_config(parallel_cpu_threads: bool) -> Result<ZebradConfig> {
+    // [Note on port conflict](#Note on port conflict)
+    let listen_port = random_known_port();
+    let listen_ip = "127.0.0.1".parse().expect("hard-coded IP is valid");
+    let zebra_rpc_listener = SocketAddr::new(listen_ip, listen_port);
+
+    // Write a configuration that has the rpc listen_addr option set
+    // TODO: split this config into another function?
+    let mut config = default_test_config()?;
+    config.rpc.listen_addr = Some(zebra_rpc_listener);
+    if parallel_cpu_threads {
+        // Auto-configure to the number of CPU cores: most users configre this
+        config.rpc.parallel_cpu_threads = 0;
+    } else {
+        // Default config, users who want to detect port conflicts configure this
+        config.rpc.parallel_cpu_threads = 1;
+    }
+
+    Ok(config)
+}

zebrad/tests/common/get_block_template_rpcs.rs

@@ -1,5 +1,3 @@
 //! Acceptance tests for getblocktemplate RPC methods in Zebra.
 
-use super::*;
-
 pub(crate) mod submit_block;