diff --git a/Cargo.lock b/Cargo.lock index 67a39732198..36c6339a5a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3941,11 +3941,13 @@ dependencies = [ "futures", "hex", "indexmap", + "lazy_static", "metrics", "pin-project 0.4.27", "proptest", "proptest-derive", "rand 0.7.3", + "regex", "serde", "thiserror", "tokio 0.3.6", @@ -3991,6 +3993,7 @@ dependencies = [ "primitive-types", "proptest", "proptest-derive", + "regex", "rlimit", "rocksdb", "serde", diff --git a/zebra-network/Cargo.toml b/zebra-network/Cargo.toml index 05ca3134c17..6117372bd7a 100644 --- a/zebra-network/Cargo.toml +++ b/zebra-network/Cargo.toml @@ -16,8 +16,10 @@ hex = "0.4" # indexmap has rayon support for parallel iteration, # which we don't use, so disable it to drop the dependencies. indexmap = { version = "1.6", default-features = false } +lazy_static = "1.4.0" pin-project = "0.4" rand = "0.7" +regex = "1" serde = { version = "1", features = ["serde_derive"] } thiserror = "1" diff --git a/zebra-network/src/constants.rs b/zebra-network/src/constants.rs index 2e089442b03..192a95cf22d 100644 --- a/zebra-network/src/constants.rs +++ b/zebra-network/src/constants.rs @@ -2,6 +2,9 @@ use std::time::Duration; +use lazy_static::lazy_static; +use regex::Regex; + // XXX should these constants be split into protocol also? use crate::protocol::external::types::*; @@ -95,6 +98,16 @@ pub const EWMA_DEFAULT_RTT: Duration = Duration::from_secs(20 + 1); /// better peers when we restart the sync. pub const EWMA_DECAY_TIME: Duration = Duration::from_secs(200); +lazy_static! { + /// OS-specific error when the port attempting to be opened is already in use. + pub static ref PORT_IN_USE_ERROR: Regex = if cfg!(unix) { + #[allow(clippy::trivial_regex)] + Regex::new("already in use") + } else { + Regex::new("(access a socket in a way forbidden by its access permissions)|(Only one usage of each socket address)") + }.expect("regex is valid"); +} + /// Magic numbers used to identify different Zcash networks. pub mod magics { use super::*; diff --git a/zebra-network/src/lib.rs b/zebra-network/src/lib.rs index 7bce06115a5..75f15ffdf35 100644 --- a/zebra-network/src/lib.rs +++ b/zebra-network/src/lib.rs @@ -66,7 +66,7 @@ pub type BoxError = Box; mod address_book; mod config; -mod constants; +pub mod constants; mod isolated; mod meta_addr; mod peer; diff --git a/zebra-network/src/peer_set/initialize.rs b/zebra-network/src/peer_set/initialize.rs index d2c8b9a39ad..e6ad0cf09d7 100644 --- a/zebra-network/src/peer_set/initialize.rs +++ b/zebra-network/src/peer_set/initialize.rs @@ -115,16 +115,7 @@ where ); let peer_set = Buffer::new(BoxService::new(peer_set), constants::PEERSET_BUFFER_SIZE); - // Connect the tx end to the 3 peer sources: - - // 1. Initial peers, specified in the config. - let add_guard = tokio::spawn(add_initial_peers( - config.initial_peers(), - connector.clone(), - peerset_tx.clone(), - )); - - // 2. Incoming peer connections, via a listener. + // 1. Incoming peer connections, via a listener. // Warn if we're configured using the wrong network port. // TODO: use the right port if the port is unspecified @@ -144,6 +135,18 @@ where let listen_guard = tokio::spawn(listen(config.listen_addr, listener, peerset_tx.clone())); + let initial_peers_fut = { + let initial_peers = config.initial_peers(); + let connector = connector.clone(); + let tx = peerset_tx.clone(); + + // Connect the tx end to the 3 peer sources: + add_initial_peers(initial_peers, connector, tx) + }; + + // 2. Initial peers, specified in the config. + let add_guard = tokio::spawn(initial_peers_fut); + // 3. Outgoing peers we connect to in response to load. let mut candidates = CandidateSet::new(address_book.clone(), peer_set.clone()); @@ -211,7 +214,18 @@ where S: Service<(TcpStream, SocketAddr), Response = peer::Client, Error = BoxError> + Clone, S::Future: Send + 'static, { - let listener = TcpListener::bind(addr).await?; + let listener_result = TcpListener::bind(addr).await; + + let listener = match listener_result { + Ok(l) => l, + Err(e) => panic!( + "Opening Zcash network protocol listener {:?} failed: {:?}. \ + Hint: Check if another zebrad or zcashd process is running. \ + Try changing the network listen_addr in the Zebra config.", + addr, e, + ), + }; + let local_addr = listener.local_addr()?; info!("Opened Zcash protocol endpoint at {}", local_addr); loop { diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index 40571b5a486..9d9259f4e20 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -13,6 +13,7 @@ zebra-chain = { path = "../zebra-chain" } dirs = "3.0.1" hex = "0.4.2" lazy_static = "1.4.0" +regex = "1" serde = { version = "1", features = ["serde_derive"] } futures = "0.3.12" diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index d99c14205a0..e51ac559a0d 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -1,3 +1,5 @@ +//! Definitions of constants. + /// The maturity threshold for transparent coinbase outputs. /// /// A transaction MUST NOT spend a transparent output of a coinbase transaction @@ -13,3 +15,11 @@ pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1; /// The database format version, incremented each time the database format changes. pub const DATABASE_FORMAT_VERSION: u32 = 4; + +use lazy_static::lazy_static; +use regex::Regex; + +lazy_static! { + /// Regex that matches the RocksDB error when its lock file is already open. + pub static ref LOCK_FILE_ERROR: Regex = Regex::new("(lock file).*(temporarily unavailable)|(in use)|(being used by another process)").expect("regex is valid"); +} diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 545b7fe375f..dc302c77d04 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -16,7 +16,7 @@ #![allow(clippy::unnecessary_wraps)] mod config; -mod constants; +pub mod constants; mod error; mod request; mod response; diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 64397851468..b9932174be7 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -41,8 +41,21 @@ impl FinalizedState { rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()), ]; - let db = rocksdb::DB::open_cf_descriptors(&db_options, path, column_families) - .expect("database path and options are valid"); + let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families); + + let db = match db_result { + Ok(d) => { + tracing::info!("Opened Zebra state cache at {}", path.display()); + d + } + // TODO: provide a different hint if the disk is full, see #1623 + Err(e) => panic!( + "Opening database {:?} failed: {:?}. \ + Hint: Check if another zebrad process is running. \ + Try changing the state cache_dir in the Zebra config.", + path, e, + ), + }; let new_state = Self { queued_by_prev_hash: HashMap::new(), diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs index e48bc993470..8395c03a37c 100644 --- a/zebra-test/src/command.rs +++ b/zebra-test/src/command.rs @@ -12,7 +12,7 @@ use std::{ io::BufRead, io::{BufReader, Lines, Read}, path::Path, - process::{Child, ChildStdout, Command, ExitStatus, Output, Stdio}, + process::{Child, ChildStderr, ChildStdout, Command, ExitStatus, Output, Stdio}, time::{Duration, Instant}, }; @@ -86,6 +86,7 @@ impl CommandExt for Command { dir, deadline: None, stdout: None, + stderr: None, bypass_test_capture: false, }) } @@ -152,6 +153,7 @@ pub struct TestChild { pub cmd: String, pub child: Child, pub stdout: Option>>, + pub stderr: Option>>, pub deadline: Option, bypass_test_capture: bool, } @@ -184,7 +186,7 @@ impl TestChild { }) } - /// Set a timeout for `expect_stdout`. + /// Set a timeout for `expect_stdout` or `expect_stderr`. /// /// Does not apply to `wait_with_output`. pub fn with_timeout(mut self, timeout: Duration) -> Self { @@ -192,17 +194,18 @@ impl TestChild { self } - /// Configures testrunner to forward stdout to the true stdout rather than - /// fakestdout used by cargo tests. + /// Configures testrunner to forward stdout and stderr to the true stdout, + /// rather than the fakestdout used by cargo tests. pub fn bypass_test_capture(mut self, cond: bool) -> Self { self.bypass_test_capture = cond; self } - /// Checks each line of the child's stdout against `regex`, and returns matching lines. + /// Checks each line of the child's stdout against `regex`, and returns Ok + /// if a line matches. /// /// Kills the child after the configured timeout has elapsed. - /// Note: the timeout is only checked after each line. + /// See `expect_line_matching` for details. #[instrument(skip(self))] pub fn expect_stdout(&mut self, regex: &str) -> Result<&mut Self> { if self.stdout.is_none() { @@ -214,12 +217,67 @@ impl TestChild { .map(BufRead::lines) } - let re = regex::Regex::new(regex).expect("regex must be valid"); let mut lines = self .stdout .take() .expect("child must capture stdout to call expect_stdout"); + match self.expect_line_matching(&mut lines, regex, "stdout") { + Ok(()) => { + self.stdout = Some(lines); + Ok(self) + } + Err(report) => Err(report), + } + } + + /// Checks each line of the child's stderr against `regex`, and returns Ok + /// if a line matches. + /// + /// Kills the child after the configured timeout has elapsed. + /// See `expect_line_matching` for details. + #[instrument(skip(self))] + pub fn expect_stderr(&mut self, regex: &str) -> Result<&mut Self> { + if self.stderr.is_none() { + self.stderr = self + .child + .stderr + .take() + .map(BufReader::new) + .map(BufRead::lines) + } + + let mut lines = self + .stderr + .take() + .expect("child must capture stderr to call expect_stderr"); + + match self.expect_line_matching(&mut lines, regex, "stderr") { + Ok(()) => { + self.stderr = Some(lines); + Ok(self) + } + Err(report) => Err(report), + } + } + + /// Checks each line in `lines` against `regex`, and returns Ok if a line + /// matches. Uses `stream_name` as the name for `lines` in error reports. + /// + /// Kills the child after the configured timeout has elapsed. + /// Note: the timeout is only checked after each full line is received from + /// the child. + #[instrument(skip(self, lines))] + pub fn expect_line_matching( + &mut self, + lines: &mut L, + regex: &str, + stream_name: &str, + ) -> Result<()> + where + L: Iterator>, + { + let re = regex::Regex::new(regex).expect("regex must be valid"); while !self.past_deadline() && self.is_running() { let line = if let Some(line) = lines.next() { line? @@ -227,20 +285,21 @@ impl TestChild { break; }; - // since we're about to discard this line write it to stdout so our - // test runner can capture it and display if the test fails, may - // cause weird reordering for stdout / stderr - if !self.bypass_test_capture { - println!("{}", line); - } else { + // Since we're about to discard this line write it to stdout, so it + // can be preserved. May cause weird reordering for stdout / stderr. + // Uses stdout even if the original lines were from stderr. + if self.bypass_test_capture { + // send lines to the terminal (or process stdout file redirect) use std::io::Write; #[allow(clippy::explicit_write)] writeln!(std::io::stdout(), "{}", line).unwrap(); + } else { + // if the test fails, the test runner captures and displays it + println!("{}", line); } if re.is_match(&line) { - self.stdout = Some(lines); - return Ok(self); + return Ok(()); } } @@ -251,9 +310,12 @@ impl TestChild { self.kill()?; } - let report = eyre!("stdout of command did not contain any matches for the given regex") - .context_from(self) - .with_section(|| format!("{:?}", regex).header("Match Regex:")); + let report = eyre!( + "{} of command did not contain any matches for the given regex", + stream_name + ) + .context_from(self) + .with_section(|| format!("{:?}", regex).header("Match Regex:")); Err(report) } @@ -340,6 +402,51 @@ impl TestOutput { .with_section(|| format!("{:?}", regex).header("Match Regex:")) } + #[instrument(skip(self))] + pub fn stderr_contains(&self, regex: &str) -> Result<&Self> { + let re = regex::Regex::new(regex)?; + let stderr = String::from_utf8_lossy(&self.output.stderr); + + for line in stderr.lines() { + if re.is_match(line) { + return Ok(self); + } + } + + Err(eyre!( + "stderr of command did not contain any matches for the given regex" + )) + .context_from(self) + .with_section(|| format!("{:?}", regex).header("Match Regex:")) + } + + #[instrument(skip(self))] + pub fn stderr_equals(&self, s: &str) -> Result<&Self> { + let stderr = String::from_utf8_lossy(&self.output.stderr); + + if stderr == s { + return Ok(self); + } + + Err(eyre!("stderr of command is not equal the given string")) + .context_from(self) + .with_section(|| format!("{:?}", s).header("Match String:")) + } + + #[instrument(skip(self))] + pub fn stderr_matches(&self, regex: &str) -> Result<&Self> { + let re = regex::Regex::new(regex)?; + let stderr = String::from_utf8_lossy(&self.output.stderr); + + if re.is_match(&stderr) { + return Ok(self); + } + + Err(eyre!("stderr of command is not equal to the given regex")) + .context_from(self) + .with_section(|| format!("{:?}", regex).header("Match Regex:")) + } + /// Returns Ok if the program was killed, Err(Report) if exit was by another /// reason. pub fn assert_was_killed(&self) -> Result<()> { @@ -423,7 +530,12 @@ impl ContextFrom<&mut TestChild> for Report { let _ = stdout.read_to_string(&mut stdout_buf); } - if let Some(stderr) = &mut source.child.stderr { + if let Some(stderr) = &mut source.stderr { + for line in stderr { + let line = if let Ok(line) = line { line } else { break }; + let _ = writeln!(&mut stderr_buf, "{}", line); + } + } else if let Some(stderr) = &mut source.child.stderr { let _ = stderr.read_to_string(&mut stderr_buf); } diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index 0fe458dc6ad..a6213df019a 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -12,6 +12,9 @@ use abscissa_core::{ use application::fatal_error; use std::process; +use zebra_network::constants::PORT_IN_USE_ERROR; +use zebra_state::constants::LOCK_FILE_ERROR; + /// Application state pub static APPLICATION: AppCell = AppCell::new(); @@ -171,7 +174,21 @@ impl Application for ZebradApp { .panic_section(metadata_section) .issue_url(concat!(env!("CARGO_PKG_REPOSITORY"), "/issues/new")) .issue_filter(|kind| match kind { - color_eyre::ErrorKind::NonRecoverable(_) => true, + color_eyre::ErrorKind::NonRecoverable(error) => { + let error_str = match error.downcast_ref::() { + Some(as_string) => as_string, + None => return true, + }; + // listener port conflicts + if PORT_IN_USE_ERROR.is_match(error_str) { + return false; + } + // RocksDB lock file conflicts + if LOCK_FILE_ERROR.is_match(error_str) { + return false; + } + true + } color_eyre::ErrorKind::Recoverable(error) => { // type checks should be faster than string conversions if error.is::() diff --git a/zebrad/src/components/metrics.rs b/zebrad/src/components/metrics.rs index 83963f3c731..15b6a22c7d1 100644 --- a/zebrad/src/components/metrics.rs +++ b/zebrad/src/components/metrics.rs @@ -12,11 +12,21 @@ impl MetricsEndpoint { /// Create the component. pub fn new(config: &ZebradConfig) -> Result { if let Some(addr) = config.metrics.endpoint_addr { - info!("Initializing metrics endpoint at {}", addr); - metrics_exporter_prometheus::PrometheusBuilder::new() + let endpoint_result = metrics_exporter_prometheus::PrometheusBuilder::new() .listen_address(addr) - .install() - .expect("FIXME ERROR CONVERSION"); + .install(); + match endpoint_result { + Ok(endpoint) => { + info!("Opened metrics endpoint at {}", addr); + endpoint + } + Err(e) => panic!( + "Opening metrics endpoint listener {:?} failed: {:?}. \ + Hint: Check if another zebrad or zcashd process is running. \ + Try changing the metrics endpoint_addr in the Zebra config.", + addr, e, + ), + } } Ok(Self {}) } diff --git a/zebrad/src/components/tracing/endpoint.rs b/zebrad/src/components/tracing/endpoint.rs index 353bd0ba60a..fe087d3c2f7 100644 --- a/zebrad/src/components/tracing/endpoint.rs +++ b/zebrad/src/components/tracing/endpoint.rs @@ -41,7 +41,6 @@ impl TracingEndpoint { } else { return Ok(()); }; - info!("Initializing tracing endpoint at {}", addr); let service = make_service_fn(|_| async { Ok::<_, hyper::Error>(service_fn(request_handler)) }); @@ -54,12 +53,16 @@ impl TracingEndpoint { // try_bind uses the tokio runtime, so we // need to construct it inside the task. let server = match Server::try_bind(&addr) { - Ok(s) => s, - Err(e) => { - error!("Could not open tracing endpoint listener"); - error!("Error: {}", e); - return; + Ok(s) => { + info!("Opened tracing endpoint at {}", addr); + s } + Err(e) => panic!( + "Opening tracing endpoint listener {:?} failed: {:?}. \ + Hint: Check if another zebrad or zcashd process is running. \ + Try changing the tracing endpoint_addr in the Zebra config.", + addr, e, + ), } .serve(service); diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 1382a66a5bf..30afd04f9d1 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -2,6 +2,7 @@ //! output for given argument combinations matches what is expected. //! //! ### Note on port conflict +//! //! If the test child has a cache or port conflict with another test, or a //! running zebrad or zcashd, then it will panic. But the acceptance tests //! expect it to run until it is killed. @@ -29,6 +30,8 @@ use zebra_chain::{ NetworkUpgrade, }, }; +use zebra_network::constants::PORT_IN_USE_ERROR; +use zebra_state::constants::LOCK_FILE_ERROR; use zebra_test::{command::TestDirExt, prelude::*}; use zebrad::config::ZebradConfig; @@ -975,7 +978,7 @@ async fn metrics_endpoint() -> Result<()> { let output = output.assert_failure()?; // Make sure metrics was started - output.stdout_contains(format!(r"Initializing metrics endpoint at {}", endpoint).as_str())?; + output.stdout_contains(format!(r"Opened metrics endpoint at {}", endpoint).as_str())?; // [Note on port conflict](#Note on port conflict) output @@ -1042,7 +1045,7 @@ async fn tracing_endpoint() -> Result<()> { let output = output.assert_failure()?; // Make sure tracing endpoint was started - output.stdout_contains(format!(r"Initializing tracing endpoint at {}", endpoint).as_str())?; + output.stdout_contains(format!(r"Opened tracing endpoint at {}", endpoint).as_str())?; // Todo: Match some trace level messages from output // [Note on port conflict](#Note on port conflict) @@ -1052,3 +1055,179 @@ async fn tracing_endpoint() -> Result<()> { Ok(()) } + +/// Test will start 2 zebrad nodes one after the other using the same Zcash listener. +/// It is expected that the first node spawned will get exclusive use of the port. +/// The second node will panic with the Zcash listener conflict hint added in #1535. +#[test] +fn zcash_listener_conflict() -> Result<()> { + zebra_test::init(); + + // [Note on port conflict](#Note on port conflict) + let port = random_known_port(); + let listen_addr = format!("127.0.0.1:{}", port); + + // Write a configuration that has our created network listen_addr + let mut config = default_test_config()?; + config.network.listen_addr = listen_addr.parse().unwrap(); + let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + let regex1 = format!(r"Opened Zcash protocol endpoint at {}", listen_addr); + + // From another folder create a configuration with the same listener. + // `network.listen_addr` will be the same in the 2 nodes. + // (But since the config is ephemeral, they will have different state paths.) + let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + + check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?; + + Ok(()) +} + +/// Start 2 zebrad nodes using the same metrics listener port, but different +/// state directories and Zcash listener ports. The first node should get +/// exclusive use of the port. The second node will panic with the Zcash metrics +/// conflict hint added in #1535. +#[test] +fn zcash_metrics_conflict() -> Result<()> { + zebra_test::init(); + + // [Note on port conflict](#Note on port conflict) + let port = random_known_port(); + let listen_addr = format!("127.0.0.1:{}", port); + + // Write a configuration that has our created metrics endpoint_addr + let mut config = default_test_config()?; + config.metrics.endpoint_addr = Some(listen_addr.parse().unwrap()); + let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + let regex1 = format!(r"Opened metrics endpoint at {}", listen_addr); + + // From another folder create a configuration with the same endpoint. + // `metrics.endpoint_addr` will be the same in the 2 nodes. + // But they will have different Zcash listeners (auto port) and states (ephemeral) + let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + + check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?; + + Ok(()) +} + +/// Start 2 zebrad nodes using the same tracing listener port, but different +/// state directories and Zcash listener ports. The first node should get +/// exclusive use of the port. The second node will panic with the Zcash tracing +/// conflict hint added in #1535. +#[test] +fn zcash_tracing_conflict() -> Result<()> { + zebra_test::init(); + + // [Note on port conflict](#Note on port conflict) + let port = random_known_port(); + let listen_addr = format!("127.0.0.1:{}", port); + + // Write a configuration that has our created tracing endpoint_addr + let mut config = default_test_config()?; + config.tracing.endpoint_addr = Some(listen_addr.parse().unwrap()); + let dir1 = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + let regex1 = format!(r"Opened tracing endpoint at {}", listen_addr); + + // From another folder create a configuration with the same endpoint. + // `tracing.endpoint_addr` will be the same in the 2 nodes. + // But they will have different Zcash listeners (auto port) and states (ephemeral) + let dir2 = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + + check_config_conflict(dir1, regex1.as_str(), dir2, PORT_IN_USE_ERROR.as_str())?; + + Ok(()) +} + +/// Start 2 zebrad nodes using the same state directory, but different Zcash +/// listener ports. The first node should get exclusive access to the database. +/// The second node will panic with the Zcash state conflict hint added in #1535. +#[test] +fn zcash_state_conflict() -> Result<()> { + zebra_test::init(); + + // A persistent config has a fixed temp state directory, but asks the OS to + // automatically choose an unused port + let mut config = persistent_test_config()?; + let dir_conflict = TempDir::new("zebrad_tests")?.with_config(&mut config)?; + + // Windows problems with this match will be worked on at #1654 + // We are matching the whole opened path only for unix by now. + let regex = if cfg!(unix) { + let mut dir_conflict_full = PathBuf::new(); + dir_conflict_full.push(dir_conflict.path()); + dir_conflict_full.push("state"); + dir_conflict_full.push("state"); + dir_conflict_full.push(format!( + "v{}", + zebra_state::constants::DATABASE_FORMAT_VERSION + )); + dir_conflict_full.push(config.network.network.to_string().to_lowercase()); + format!( + "Opened Zebra state cache at {}", + dir_conflict_full.display() + ) + } else { + String::from("Opened Zebra state cache at ") + }; + + check_config_conflict( + dir_conflict.path(), + regex.as_str(), + dir_conflict.path(), + LOCK_FILE_ERROR.as_str(), + )?; + + Ok(()) +} + +/// Launch a node in `first_dir`, wait a few seconds, then launch a node in +/// `second_dir`. Check that the first node's stdout contains +/// `first_stdout_regex`, and the second node's stderr contains +/// `second_stderr_regex`. +fn check_config_conflict( + first_dir: T, + first_stdout_regex: &str, + second_dir: U, + second_stderr_regex: &str, +) -> Result<()> +where + T: ZebradTestDirExt, + U: ZebradTestDirExt, +{ + // By DNS issues we want to skip all port conflict tests on macOS by now. + // Follow up at #1631 + if cfg!(target_os = "macos") { + return Ok(()); + } + + // Start the first node + let mut node1 = first_dir.spawn_child(&["start"])?; + + // Wait a bit to spawn the second node, we want the first fully started. + std::thread::sleep(LAUNCH_DELAY); + + // Spawn the second node + let node2 = second_dir.spawn_child(&["start"])?; + + // Wait a few seconds and kill first node. + // Second node is terminated by panic, no need to kill. + std::thread::sleep(LAUNCH_DELAY); + node1.kill()?; + + // In node1 we want to check for the success regex + let output1 = node1.wait_with_output()?; + output1.stdout_contains(first_stdout_regex)?; + output1 + .assert_was_killed() + .wrap_err("Possible port conflict. Are there other acceptance tests running?")?; + + // In the second node we look for the conflict regex + let output2 = node2.wait_with_output()?; + output2.stderr_contains(second_stderr_regex)?; + output2 + .assert_was_not_killed() + .wrap_err("Possible port conflict. Are there other acceptance tests running?")?; + + Ok(()) +}