diff --git a/CHANGELOG.md b/CHANGELOG.md index c245308b39..56dc27118e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ # UNRELEASED +### feat: `dfx start` for the shared local network stores replica state files in unique directories by options + +The state files for different replica versions are often incompatible, +so `dfx start` requires the `--clean` argument in order to reset data when +using different replica versions or different replica options. + +For the local shared network, dfx now stores replica state files in different +directories, split up by replica version and options. + +As an example, you'll be able to do things like this going forward: +```bash +dfx +0.21.0 start +(cd project1 && dfx deploy && dfx canister call ...) +dfx stop + +dfx +0.22.0 start +# notice --clean is not required. +# even if --clean were passed, the canisters for project1 would be unaffected. +(cd project2 && dfx deploy) +# project1 won't be affected unless you call dfx in its directory +dfx stop + +dfx +0.21.0 start +# the canisters are still deployed +(cd project1 && dfx canister call ...) +``` + +Prior to this change, the second `dfx start` would have had to include `--clean`, +which would have reset the state of the shared local network, affecting all projects. + +This also means `dfx start` for the shared local network won't ever require you to pass `--clean`. + +`dfx start` will delete old replica state directories. At present, it retains the 10 most recently used. + +This doesn't apply to project-specific networks, and it doesn't apply with `--pocketic`. + +It doesn't apply to project-specific networks because the project's canister ids would +reset anyway on first access. If you run `dfx start` in a project directory where dfx.json +defines the local network, you'll still be prompted to run with `--clean` if using a +different replica version or different replica options. + +It doesn't apply to `--pocketic` because PocketIC does not yet persist any data. + # 0.20.2 ### fix: `dfx canister delete` fails diff --git a/Cargo.lock b/Cargo.lock index 320651d790..2f4f31b7b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,6 +1617,7 @@ dependencies = [ "semver", "serde", "serde_json", + "sha2 0.10.8", "slog", "tar", "tempfile", diff --git a/e2e/tests-dfx/shared_local_network.bash b/e2e/tests-dfx/shared_local_network.bash index 52044fee2f..053d63be1d 100644 --- a/e2e/tests-dfx/shared_local_network.bash +++ b/e2e/tests-dfx/shared_local_network.bash @@ -12,6 +12,16 @@ teardown() { standard_teardown } +@test "shared local network always requires --clean with pocketic" { + [[ "$USE_POCKETIC" ]] || skip "specific to dfx start --pocketic" + + assert_command dfx_start + assert_command dfx stop + + assert_command_fail dfx start --pocketic + assert_contains "The network state can't be reused with this configuration. Rerun with \`--clean\`" +} + @test "dfx start creates no files in the current directory when run from an empty directory" { dfx_start assert_command find . diff --git a/e2e/tests-dfx/start.bash b/e2e/tests-dfx/start.bash index 9aae68219a..53069fa858 100644 --- a/e2e/tests-dfx/start.bash +++ b/e2e/tests-dfx/start.bash @@ -12,6 +12,45 @@ teardown() { standard_teardown } +@test "start and stop with different options" { + [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic: artificial delay, and clean required" + dfx_start --artificial-delay 101 + dfx_stop + + # notice: no need to --clean + dfx_start --artificial-delay 102 + dfx_stop +} + +@test "project networks still need --clean" { + [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic: artificial delay" + dfx_new hello + define_project_network + + dfx_start --artificial-delay 101 + dfx stop + + assert_command_fail dfx_start --artificial-delay 102 +} + +@test "stop and start with other options does not disrupt projects" { + [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic: artificial delay" + dfx_start --artificial-delay 101 + + dfx_new p1 + assert_command dfx deploy + CANISTER_ID="$(dfx canister id p1_backend)" + + assert_command dfx stop + dfx_start --artificial-delay 102 + assert_command dfx stop + + dfx_start --artificial-delay 101 + + assert_command dfx canister id p1_backend + assert_eq "$CANISTER_ID" +} + @test "start and stop outside project" { dfx_start @@ -447,18 +486,14 @@ teardown() { assert_match "Hello, World! from DFINITY" } -@test "modifying networks.json requires --clean on restart" { +@test "modifying networks.json does not require --clean on restart" { [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic: --force" dfx_start dfx stop assert_command dfx_start dfx stop jq -n '.local.replica.log_level="warning"' > "$E2E_NETWORKS_JSON" - assert_command_fail dfx_start - assert_contains "The network state can't be reused with this configuration. Rerun with \`--clean\`." - assert_command dfx_start --force - dfx stop - assert_command dfx_start --clean + assert_command dfx_start } @test "project-local networks require --clean if dfx.json was updated" { @@ -480,8 +515,11 @@ teardown() { assert_command dfx_start --clean } -@test "flags count as configuration modification and require --clean" { +@test "flags count as configuration modification and require --clean for a project network" { [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic: --artificial-delay" + dfx_new + define_project_network + dfx start --background dfx stop assert_command_fail dfx start --artificial-delay 100 --background diff --git a/src/dfx-core/Cargo.toml b/src/dfx-core/Cargo.toml index 0b189ecca3..680b674f52 100644 --- a/src/dfx-core/Cargo.toml +++ b/src/dfx-core/Cargo.toml @@ -35,6 +35,7 @@ sec1 = { workspace = true, features = ["std"] } semver = { workspace = true, features = ["serde"] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true slog = { workspace = true, features = ["max_level_trace"] } tar.workspace = true tempfile.workspace = true diff --git a/src/dfx-core/src/config/model/local_server_descriptor.rs b/src/dfx-core/src/config/model/local_server_descriptor.rs index 31beb4b759..9eb5cd6ad9 100644 --- a/src/dfx-core/src/config/model/local_server_descriptor.rs +++ b/src/dfx-core/src/config/model/local_server_descriptor.rs @@ -30,6 +30,7 @@ pub struct LocalServerDescriptor { /// $HOME/.local/share/dfx/network/local /// $APPDATA/dfx/network/local pub data_directory: PathBuf, + pub settings_digest: Option, pub bind_address: SocketAddr, @@ -62,9 +63,11 @@ impl LocalServerDescriptor { scope: LocalNetworkScopeDescriptor, legacy_pid_path: Option, ) -> Result { + let settings_digest = None; let bind_address = to_socket_addr(&bind).map_err(ParseBindAddressFailed)?; Ok(LocalServerDescriptor { data_directory, + settings_digest, bind_address, bitcoin, canister_http, @@ -78,7 +81,7 @@ impl LocalServerDescriptor { /// The contents of this file are different for each `dfx start --clean` /// or `dfx start` when the network data directory doesn't already exist pub fn network_id_path(&self) -> PathBuf { - self.data_directory.join("network-id") + self.data_dir_by_settings_digest().join("network-id") } /// This file contains the pid of the process started with `dfx start` @@ -164,9 +167,21 @@ impl LocalServerDescriptor { path.exists().then(|| load_json_file(&path)).transpose() } + pub fn data_dir_by_settings_digest(&self) -> PathBuf { + if self.scope == LocalNetworkScopeDescriptor::Project { + self.data_directory.clone() + } else { + let settings_digest = self + .settings_digest + .as_ref() + .expect("settings_digest must be set"); + self.data_directory.join(settings_digest) + } + } + /// The top-level directory holding state for the replica. pub fn state_dir(&self) -> PathBuf { - self.data_directory.join("state") + self.data_dir_by_settings_digest().join("state") } /// The replicated state of the replica. @@ -184,6 +199,11 @@ impl LocalServerDescriptor { pub fn effective_config_path(&self) -> PathBuf { self.data_directory.join("replica-effective-config.json") } + + pub fn effective_config_path_by_settings_digest(&self) -> PathBuf { + self.data_dir_by_settings_digest() + .join("replica-effective-config.json") + } } impl LocalServerDescriptor { @@ -224,6 +244,13 @@ impl LocalServerDescriptor { }; Self { proxy, ..self } } + + pub fn with_settings_digest(self, settings_digest: String) -> Self { + Self { + settings_digest: Some(settings_digest), + ..self + } + } } impl LocalServerDescriptor { diff --git a/src/dfx-core/src/config/model/mod.rs b/src/dfx-core/src/config/model/mod.rs index 413203fa8f..2014e4ae34 100644 --- a/src/dfx-core/src/config/model/mod.rs +++ b/src/dfx-core/src/config/model/mod.rs @@ -6,3 +6,4 @@ pub mod extension_canister_type; pub mod local_server_descriptor; pub mod network_descriptor; pub mod replica_config; +pub mod settings_digest; diff --git a/src/dfx-core/src/config/model/replica_config.rs b/src/dfx-core/src/config/model/replica_config.rs index b36ee42932..297dec29d1 100644 --- a/src/dfx-core/src/config/model/replica_config.rs +++ b/src/dfx-core/src/config/model/replica_config.rs @@ -184,7 +184,7 @@ impl HttpHandlerConfig { } } -#[derive(Serialize, Deserialize, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] #[serde(tag = "type", rename_all = "snake_case")] #[allow(clippy::large_enum_variant)] pub enum CachedReplicaConfig<'a> { @@ -192,7 +192,7 @@ pub enum CachedReplicaConfig<'a> { PocketIc, } -#[derive(Serialize, Deserialize, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] pub struct CachedConfig<'a> { pub replica_rev: String, #[serde(flatten)] diff --git a/src/dfx-core/src/config/model/settings_digest.rs b/src/dfx-core/src/config/model/settings_digest.rs new file mode 100644 index 0000000000..5ee03a5daf --- /dev/null +++ b/src/dfx-core/src/config/model/settings_digest.rs @@ -0,0 +1,115 @@ +use crate::config::model::dfinity::{ReplicaLogLevel, ReplicaSubnetType}; +use crate::config::model::local_server_descriptor::LocalServerDescriptor; +use candid::Deserialize; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::borrow::Cow; + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "snake_case")] +enum HttpHandlerPortSetting { + Port { + port: u16, + }, + #[default] + WritePortToPath, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct HttpHandlerSettings { + pub port: HttpHandlerPortSetting, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct BtcAdapterSettings { + pub enabled: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct CanisterHttpAdapterSettings { + pub enabled: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct ReplicaSettings { + pub http_handler: HttpHandlerSettings, + pub subnet_type: ReplicaSubnetType, + pub btc_adapter: BtcAdapterSettings, + pub canister_http_adapter: CanisterHttpAdapterSettings, + pub log_level: ReplicaLogLevel, + pub artificial_delay: u32, + pub use_old_metering: bool, +} + +#[derive(Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "snake_case")] +enum BackendSettings<'a> { + Replica { settings: Cow<'a, ReplicaSettings> }, + PocketIc, +} + +#[derive(Serialize, Deserialize, PartialEq, Eq)] +struct Settings<'a> { + pub ic_repo_commit: String, + #[serde(flatten)] + pub backend: BackendSettings<'a>, +} + +pub fn get_settings_digest( + ic_repo_commit: &str, + local_server_descriptor: &LocalServerDescriptor, + use_old_metering: bool, + artificial_delay: u32, + pocketic: bool, +) -> String { + let backend = if pocketic { + BackendSettings::PocketIc + } else { + get_replica_backend_settings(local_server_descriptor, use_old_metering, artificial_delay) + }; + let settings = Settings { + ic_repo_commit: ic_repo_commit.into(), + backend, + }; + let normalized = serde_json::to_string_pretty(&settings).unwrap(); + let hash: Vec = Sha256::digest(normalized).to_vec(); + hex::encode(hash) +} + +fn get_replica_backend_settings( + local_server_descriptor: &LocalServerDescriptor, + use_old_metering: bool, + artificial_delay: u32, +) -> BackendSettings { + let http_handler = HttpHandlerSettings { + port: if let Some(port) = local_server_descriptor.replica.port { + HttpHandlerPortSetting::Port { port } + } else { + HttpHandlerPortSetting::WritePortToPath + }, + }; + let btc_adapter = BtcAdapterSettings { + enabled: local_server_descriptor.bitcoin.enabled, + }; + let canister_http_adapter = CanisterHttpAdapterSettings { + enabled: local_server_descriptor.canister_http.enabled, + }; + let replica_settings = ReplicaSettings { + http_handler, + subnet_type: local_server_descriptor + .replica + .subnet_type + .unwrap_or_default(), + btc_adapter, + canister_http_adapter, + log_level: local_server_descriptor + .replica + .log_level + .unwrap_or_default(), + artificial_delay, + use_old_metering, + }; + BackendSettings::Replica { + settings: Cow::Owned(replica_settings), + } +} diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs index 8f48ce3d01..ddb173a0e6 100644 --- a/src/dfx/src/commands/start.rs +++ b/src/dfx/src/commands/start.rs @@ -16,17 +16,21 @@ use crate::util::get_reusable_socket_addr; use actix::Recipient; use anyhow::{anyhow, bail, Context, Error}; use clap::{ArgAction, Parser}; -use dfx_core::config::model::local_server_descriptor::LocalServerDescriptor; -use dfx_core::config::model::network_descriptor::NetworkDescriptor; -use dfx_core::config::model::replica_config::{CachedConfig, ReplicaConfig}; -use dfx_core::config::model::{bitcoin_adapter, canister_http_adapter}; -use dfx_core::json::{load_json_file, save_json_file}; -use dfx_core::network::provider::{create_network_descriptor, LocalBindDetermination}; +use dfx_core::{ + config::model::{ + bitcoin_adapter, canister_http_adapter, + local_server_descriptor::{LocalNetworkScopeDescriptor, LocalServerDescriptor}, + network_descriptor::NetworkDescriptor, + replica_config::{CachedConfig, ReplicaConfig}, + settings_digest::get_settings_digest, + }, + fs, + json::{load_json_file, save_json_file}, + network::provider::{create_network_descriptor, LocalBindDetermination}, +}; use fn_error_context::context; use os_str_bytes::{OsStrBytes, OsStringBytes}; use slog::{info, warn, Logger}; -use std::fs; -use std::fs::create_dir_all; use std::io::Read; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -164,6 +168,7 @@ pub fn exec( } else { Some(env.get_logger().clone()) }; + let network_descriptor = create_network_descriptor( project_config, env.get_networks_config(), @@ -181,9 +186,13 @@ pub fn exec( bitcoin_node, enable_canister_http, domain, + use_old_metering, + artificial_delay, + pocketic, )?; let local_server_descriptor = network_descriptor.local_server_descriptor()?; + let pid_file_path = local_server_descriptor.dfx_pid_path(); check_previous_process_running(local_server_descriptor)?; @@ -196,18 +205,25 @@ pub fn exec( let (frontend_url, address_and_port) = frontend_address(local_server_descriptor, background)?; - let network_temp_dir = local_server_descriptor.data_directory.clone(); - create_dir_all(&network_temp_dir).with_context(|| { - format!( - "Failed to create network temp directory {}.", - network_temp_dir.to_string_lossy() - ) - })?; + fs::create_dir_all(&local_server_descriptor.data_dir_by_settings_digest())?; if !local_server_descriptor.network_id_path().exists() { write_network_id(local_server_descriptor)?; } + if let LocalNetworkScopeDescriptor::Shared { network_id_path } = &local_server_descriptor.scope + { + fs::copy(&local_server_descriptor.network_id_path(), network_id_path)?; + let effective_config_path_by_settings_digest = + local_server_descriptor.effective_config_path_by_settings_digest(); + if effective_config_path_by_settings_digest.exists() { + fs::copy( + &effective_config_path_by_settings_digest, + &local_server_descriptor.effective_config_path(), + )?; + } + } + clean_older_state_dirs(local_server_descriptor)?; let state_root = local_server_descriptor.state_dir(); let btc_adapter_socket_holder_path = local_server_descriptor.btc_adapter_socket_holder_path(); @@ -232,12 +248,7 @@ pub fn exec( // dfx info replica-port will read these port files to find out which port to use, // so we need to make sure only one has a valid port in it. let replica_config_dir = local_server_descriptor.replica_configuration_dir(); - fs::create_dir_all(&replica_config_dir).with_context(|| { - format!( - "Failed to create replica config directory {}.", - replica_config_dir.display() - ) - })?; + fs::create_dir_all(&replica_config_dir)?; let replica_port_path = empty_writable_path(local_server_descriptor.replica_port_path())?; let pocketic_port_path = empty_writable_path(local_server_descriptor.pocketic_port_path())?; @@ -259,14 +270,7 @@ pub fn exec( local_server_descriptor.describe(env.get_logger()); write_pid(&pid_file_path); - std::fs::write(&webserver_port_path, address_and_port.port().to_string()).with_context( - || { - format!( - "Failed to write webserver port file {}.", - webserver_port_path.to_string_lossy() - ) - }, - )?; + fs::write(&webserver_port_path, address_and_port.port().to_string())?; let btc_adapter_config = configure_btc_adapter_if_enabled( local_server_descriptor, @@ -330,7 +334,16 @@ pub fn exec( CachedConfig::replica(&replica_config, replica_rev().into()) }; - if !clean && !force && previous_config_path.exists() { + let is_shared_network = matches!( + &local_server_descriptor.scope, + LocalNetworkScopeDescriptor::Shared { .. } + ); + if is_shared_network && !pocketic { + save_json_file( + &local_server_descriptor.effective_config_path_by_settings_digest(), + &effective_config, + )?; + } else if !clean && !force && previous_config_path.exists() { let previous_config = load_json_file(&previous_config_path) .context("Failed to read replica configuration. Rerun with `--clean`.")?; if !effective_config.can_share_state(&previous_config) { @@ -339,8 +352,7 @@ pub fn exec( ) } } - save_json_file(&previous_config_path, &effective_config) - .context("Failed to write replica configuration")?; + save_json_file(&previous_config_path, &effective_config)?; let network_descriptor = network_descriptor.clone(); @@ -420,6 +432,54 @@ pub fn exec( Ok(()) } +fn clean_older_state_dirs(local_server_descriptor: &LocalServerDescriptor) -> DfxResult { + let directories_to_keep = 10; + let settings_digest = local_server_descriptor.settings_digest.as_ref().unwrap(); + + let data_dir = &local_server_descriptor.data_directory; + if !data_dir.is_dir() { + return Ok(()); + } + let mut state_dirs = fs::read_dir(data_dir)? + .filter_map(|e| match e { + Ok(entry) if is_candidate_state_dir(&entry.path(), settings_digest) => { + Some(Ok(entry.path())) + } + Ok(_) => None, + Err(e) => Some(Err(e)), + }) + .collect::, _>>()?; + + // keep the X most recent directories + state_dirs.sort_by_cached_key(|p| { + p.metadata() + .map(|m| m.modified().unwrap_or(SystemTime::UNIX_EPOCH)) + .unwrap_or(SystemTime::UNIX_EPOCH) + }); + state_dirs = state_dirs + .iter() + .rev() + .skip(directories_to_keep) + .cloned() + .collect(); + + for dir in state_dirs { + fs::remove_dir_all(&dir)?; + } + Ok(()) +} + +fn is_candidate_state_dir(path: &Path, settings_digest: &str) -> bool { + path.is_dir() + && path + .file_name() + .map(|f| { + let filename: String = f.to_string_lossy().into(); + filename != *settings_digest + }) + .unwrap_or(true) +} + pub fn apply_command_line_parameters( logger: &Logger, network_descriptor: NetworkDescriptor, @@ -429,6 +489,9 @@ pub fn apply_command_line_parameters( bitcoin_nodes: Vec, enable_canister_http: bool, domain: Vec, + use_old_metering: bool, + artificial_delay: u32, + pocketic: bool, ) -> DfxResult { if enable_canister_http { warn!( @@ -465,6 +528,16 @@ pub fn apply_command_line_parameters( local_server_descriptor = local_server_descriptor.with_proxy_domains(domain) } + let settings_digest = get_settings_digest( + replica_rev(), + &local_server_descriptor, + use_old_metering, + artificial_delay, + pocketic, + ); + + local_server_descriptor = local_server_descriptor.with_settings_digest(settings_digest); + Ok(NetworkDescriptor { local_server_descriptor: Some(local_server_descriptor), ..network_descriptor