From e481a8b2fc4c180ac88c53de9dd30ff38f4a8b1b Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 31 Aug 2021 07:11:35 +1200 Subject: [PATCH 01/26] Add validator config struct --- cli/src/config.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 69dbd518a3..0744f99c3e 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -451,7 +451,8 @@ fn deser_programs( #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Test { - pub genesis: Vec, + pub genesis: Option>, + pub validator: Validator } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -462,6 +463,33 @@ pub struct GenesisEntry { pub program: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Validator { + // Range to use for dynamically assigned ports. + pub dynamic_port_range: Option, + // Enable the faucet on this port. + pub faucet_port: Option, + // Give the faucet address this much SOL in genesis. + pub faucet_sol: Option, + // URL for Solana's JSON RPC or moniker. + pub url: Option, + // Directory for ledger storage. + #[serde(default = "default_ledger_directory")] + pub ledger: Option, + // Keep this amount of shreds in root slots. + pub limit_ledger_size: Option, + // Enable JSON RPC on this port, and the next port for the RPC websocket. + pub rpc_port: Option, + // Override the number of slots in an epoch. + pub slots_per_epoch: Option, + // Warp the ledger to WARP_SLOT after starting the validator. + pub warp_slot: Option, +} + +fn default_ledger_directory() -> Option { + Some(".anchor/test-ledger".to_string()) +} + #[derive(Debug, Clone)] pub struct Program { pub lib_name: String, From 2c153920b6d7c23c4375143e21c5b094757380a7 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 31 Aug 2021 07:13:35 +1200 Subject: [PATCH 02/26] Update handling of optional entries --- cli/src/lib.rs | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 3cb7187d9c..88138391a1 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1398,12 +1398,15 @@ fn genesis_flags(cfg: &WithPath) -> Result> { } } if let Some(test) = cfg.test.as_ref() { - for entry in &test.genesis { - flags.push("--bpf-program".to_string()); - flags.push(entry.address.clone()); - flags.push(entry.program.clone()); + if let Some(genesis) = &test.genesis { + for entry in genesis { + flags.push("--bpf-program".to_string()); + flags.push(entry.address.clone()); + flags.push(entry.program.clone()); + } } } + Ok(flags) } @@ -1439,17 +1442,19 @@ fn stream_logs(config: &WithPath) -> Result> { handles.push(child); } if let Some(test) = config.test.as_ref() { - for entry in &test.genesis { - let log_file = File::create(format!("{}/{}.log", program_logs_dir, entry.address))?; - let stdio = std::process::Stdio::from(log_file); - let child = std::process::Command::new("solana") - .arg("logs") - .arg(entry.address.clone()) - .arg("--url") - .arg(config.provider.cluster.url()) - .stdout(stdio) - .spawn()?; - handles.push(child); + if let Some(genesis) = &test.genesis { + for entry in genesis { + let log_file = File::create(format!("{}/{}.log", program_logs_dir, entry.address))?; + let stdio = std::process::Stdio::from(log_file); + let child = std::process::Command::new("solana") + .arg("logs") + .arg(entry.address.clone()) + .arg("--url") + .arg(config.provider.cluster.url()) + .stdout(stdio) + .spawn()?; + handles.push(child); + } } } From 77a7293ca325ba3acde17cc179ea02814943cda7 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Wed, 1 Sep 2021 14:03:01 +1200 Subject: [PATCH 03/26] Skip serializing nulls and remove default test ledger directory --- cli/src/config.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 0744f99c3e..70bf55a4eb 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -452,7 +452,7 @@ fn deser_programs( #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Test { pub genesis: Option>, - pub validator: Validator + pub validator: Validator, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -466,30 +466,31 @@ pub struct GenesisEntry { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Validator { // Range to use for dynamically assigned ports. + #[serde(skip_serializing_if = "Option::is_none")] pub dynamic_port_range: Option, // Enable the faucet on this port. + #[serde(skip_serializing_if = "Option::is_none")] pub faucet_port: Option, // Give the faucet address this much SOL in genesis. + #[serde(skip_serializing_if = "Option::is_none")] pub faucet_sol: Option, // URL for Solana's JSON RPC or moniker. + #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, - // Directory for ledger storage. - #[serde(default = "default_ledger_directory")] - pub ledger: Option, // Keep this amount of shreds in root slots. + #[serde(skip_serializing_if = "Option::is_none")] pub limit_ledger_size: Option, // Enable JSON RPC on this port, and the next port for the RPC websocket. - pub rpc_port: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rpc_port: Option, // Override the number of slots in an epoch. + #[serde(skip_serializing_if = "Option::is_none")] pub slots_per_epoch: Option, // Warp the ledger to WARP_SLOT after starting the validator. + #[serde(skip_serializing_if = "Option::is_none")] pub warp_slot: Option, } -fn default_ledger_directory() -> Option { - Some(".anchor/test-ledger".to_string()) -} - #[derive(Debug, Clone)] pub struct Program { pub lib_name: String, From 4087f7bd6867fbfbaaeeca0f0839ffc62bdbec18 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Wed, 1 Sep 2021 14:03:51 +1200 Subject: [PATCH 04/26] Handle non standard port --- cli/src/lib.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 88138391a1..0000f67b93 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1308,7 +1308,7 @@ fn test( if is_localnet && (!skip_local_validator) { let flags = match skip_deploy { true => None, - false => Some(genesis_flags(cfg)?), + false => Some(validator_flags(cfg)?), }; validator_handle = Some(start_test_validator(cfg, flags)?); } @@ -1367,7 +1367,7 @@ fn test( // Returns the solana-test-validator flags to embed the workspace programs // in the genesis block. This allows us to run tests without every deploying. -fn genesis_flags(cfg: &WithPath) -> Result> { +fn validator_flags(cfg: &WithPath) -> Result> { let programs = cfg.programs.get(&Cluster::Localnet); let mut flags = Vec::new(); @@ -1397,6 +1397,7 @@ fn genesis_flags(cfg: &WithPath) -> Result> { write_idl(idl, OutFile::File(idl_out))?; } } + if let Some(test) = cfg.test.as_ref() { if let Some(genesis) = &test.genesis { for entry in genesis { @@ -1405,6 +1406,10 @@ fn genesis_flags(cfg: &WithPath) -> Result> { flags.push(entry.program.clone()); } } + for (key, value) in serde_json::to_value(&test.validator)?.as_object().unwrap() { + flags.push(format!("--{}", key.replace("_", "-"))); + flags.push(value.to_string()); + } } Ok(flags) @@ -1481,19 +1486,26 @@ fn start_test_validator(cfg: &Config, flags: Option>) -> Result flags[position + 1].clone(), + None => "8899".to_string(), + }; + // Wait for the validator to be ready. - let client = RpcClient::new("http://localhost:8899".to_string()); + let client = RpcClient::new(format!("http://localhost:{}", port)); let mut count = 0; let ms_wait = 5000; while count < ms_wait { From 4854df0133fe3aaf9e20e3c6d11a42c6794d8343 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Wed, 1 Sep 2021 14:18:22 +1200 Subject: [PATCH 05/26] Make validator key optional --- cli/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 70bf55a4eb..1e53ee4f4e 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -452,7 +452,7 @@ fn deser_programs( #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Test { pub genesis: Option>, - pub validator: Validator, + pub validator: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] From 399932ae1b602e0f706fbc465ad0201ac23509f7 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 14 Sep 2021 13:14:12 +1200 Subject: [PATCH 06/26] Any local validator is localnet --- cli/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 4ec59db737..3e1b81bbd9 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1322,7 +1322,8 @@ fn test( // 2. The cluster is localnet, but we're not booting a local validator. // // In either case, skip the deploy if the user specifies. - let is_localnet = cfg.provider.cluster == Cluster::Localnet; + let is_localnet = cfg.provider.cluster.url().contains("127.0.0.1") + || cfg.provider.cluster.url().contains("localhost"); if (!is_localnet || skip_local_validator) && !skip_deploy { deploy(cfg_override, None)?; } From a2c4f8085f1689c5f210f50b051ad4297b1fe018 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 14 Sep 2021 14:43:15 +1200 Subject: [PATCH 07/26] Add more solana-test-validator config options --- cli/src/config.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index c188424e9a..0b081289f0 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -473,22 +473,34 @@ pub struct GenesisEntry { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Validator { - // Range to use for dynamically assigned ports. + // IP address to bind the validator ports. [default: 0.0.0.0] + #[serde(skip_serializing_if = "Option::is_none")] + pub bind_address: Option, + // Range to use for dynamically assigned ports. [default: 1024-65535] #[serde(skip_serializing_if = "Option::is_none")] pub dynamic_port_range: Option, - // Enable the faucet on this port. + // Enable the faucet on this port [deafult: 9900]. #[serde(skip_serializing_if = "Option::is_none")] pub faucet_port: Option, - // Give the faucet address this much SOL in genesis. + // Give the faucet address this much SOL in genesis. [default: 1000000] #[serde(skip_serializing_if = "Option::is_none")] pub faucet_sol: Option, + // Gossip DNS name or IP address for the validator to advertise in gossip. [default: 127.0.0.1] + #[serde(skip_serializing_if = "Option::is_none")] + pub gossip_host: Option, + // Gossip port number for the validator + #[serde(skip_serializing_if = "Option::is_none")] + pub gossip_port: Option, // URL for Solana's JSON RPC or moniker. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, - // Keep this amount of shreds in root slots. + // Use DIR as ledger location + #[serde(skip_serializing_if = "Option::is_none")] + pub ledger: Option, + // Keep this amount of shreds in root slots. [default: 10000] #[serde(skip_serializing_if = "Option::is_none")] pub limit_ledger_size: Option, - // Enable JSON RPC on this port, and the next port for the RPC websocket. + // Enable JSON RPC on this port, and the next port for the RPC websocket. [default: 8899] #[serde(skip_serializing_if = "Option::is_none")] pub rpc_port: Option, // Override the number of slots in an epoch. From 567dd57847245ad938c3d8642f99a8f44f63cb06 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 14 Sep 2021 16:19:02 +1200 Subject: [PATCH 08/26] Checkpoint --- cli/src/lib.rs | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 3e1b81bbd9..db5b04eba3 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1322,19 +1322,19 @@ fn test( // 2. The cluster is localnet, but we're not booting a local validator. // // In either case, skip the deploy if the user specifies. - let is_localnet = cfg.provider.cluster.url().contains("127.0.0.1") - || cfg.provider.cluster.url().contains("localhost"); + let is_localnet = cfg.provider.cluster == Cluster::Localnet; if (!is_localnet || skip_local_validator) && !skip_deploy { deploy(cfg_override, None)?; } // Start local test validator, if needed. let mut validator_handle = None; + let mut provider_url = cfg.provider.cluster.url(); if is_localnet && (!skip_local_validator) { let flags = match skip_deploy { true => None, false => Some(validator_flags(cfg)?), }; - validator_handle = Some(start_test_validator(cfg, flags)?); + (validator_handle, provider_url) = Some(start_test_validator(cfg, flags)?); } // Setup log reader. @@ -1354,8 +1354,7 @@ fn test( let program = args.remove(0); std::process::Command::new(program) - .args(args) - .env("ANCHOR_PROVIDER_URL", cfg.provider.cluster.url()) + .env("ANCHOR_PROVIDER_URL", provider_url) .env("ANCHOR_WALLET", cfg.provider.wallet.to_string()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -1495,25 +1494,24 @@ pub struct IdlTestMetadata { address: String, } -fn start_test_validator(cfg: &Config, flags: Option>) -> Result { - fs::create_dir_all(".anchor")?; - let test_ledger_filename = ".anchor/test-ledger"; - let test_ledger_log_filename = ".anchor/test-ledger-log.txt"; - - if Path::new(test_ledger_filename).exists() { - std::fs::remove_dir_all(test_ledger_filename)?; - } - if Path::new(test_ledger_log_filename).exists() { - std::fs::remove_file(test_ledger_log_filename)?; +fn start_test_validator(cfg: &Config, flags: Option>) -> Result<(Child, String)> { + let flags = flags.unwrap_or_default(); + let test_ledger_directory: &str = match flags.iter().position(|f| *f == "--ledger") { + Some(position) => flags[position + 1].as_ref(), + None => ".anchor/test-ledger".as_ref(), + }; + if Path::new(test_ledger_directory).exists() { + std::fs::remove_dir_all(test_ledger_directory)?; } + fs::create_dir_all(test_ledger_directory)?; + let test_ledger_log_filename = format!("{}/test-ledger-log.txt", test_ledger_directory); // Start a validator for testing. let test_validator_stdout = File::create(test_ledger_log_filename)?; let test_validator_stderr = test_validator_stdout.try_clone()?; - let flags = flags.unwrap_or_default(); let validator_handle = std::process::Command::new("solana-test-validator") .arg("--ledger") - .arg(test_ledger_filename) + .arg(test_ledger_directory) .arg("--mint") .arg(cfg.wallet_kp()?.pubkey().to_string()) .args(&flags) @@ -1522,14 +1520,18 @@ fn start_test_validator(cfg: &Config, flags: Option>) -> Result flags[position + 1].clone(), + None => "127.0.0.1".to_string(), + }; // If a custom port is defined use that for validator ready check. let port = match flags.iter().position(|f| *f == "--rpc-port") { Some(position) => flags[position + 1].clone(), None => "8899".to_string(), }; - // Wait for the validator to be ready. - let client = RpcClient::new(format!("http://localhost:{}", port)); + let rpc_url = format!("http://{}:{}", bind_address, port); + let client = RpcClient::new(rpc_url); let mut count = 0; let ms_wait = 5000; while count < ms_wait { @@ -1544,8 +1546,7 @@ fn start_test_validator(cfg: &Config, flags: Option>) -> Result) -> Result<()> { From 7331a82c51bce52651ca193ba198a4e52c90abc0 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Thu, 30 Sep 2021 15:29:35 +1300 Subject: [PATCH 09/26] WIP --- cli/src/lib.rs | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index db5b04eba3..c8e3882bc4 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1334,7 +1334,8 @@ fn test( true => None, false => Some(validator_flags(cfg)?), }; - (validator_handle, provider_url) = Some(start_test_validator(cfg, flags)?); + validator_handle = Some(start_test_validator(cfg, flags)?); + provider_url = test_validator_rpc_url(flags).as_str(); } // Setup log reader. @@ -1494,7 +1495,7 @@ pub struct IdlTestMetadata { address: String, } -fn start_test_validator(cfg: &Config, flags: Option>) -> Result<(Child, String)> { +fn start_test_validator(cfg: &Config, flags: Option>) -> Result { let flags = flags.unwrap_or_default(); let test_ledger_directory: &str = match flags.iter().position(|f| *f == "--ledger") { Some(position) => flags[position + 1].as_ref(), @@ -1520,18 +1521,7 @@ fn start_test_validator(cfg: &Config, flags: Option>) -> Result<(Chi .spawn() .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; - let bind_address = match flags.iter().position(|f| *f == "--bind-address") { - Some(position) => flags[position + 1].clone(), - None => "127.0.0.1".to_string(), - }; - // If a custom port is defined use that for validator ready check. - let port = match flags.iter().position(|f| *f == "--rpc-port") { - Some(position) => flags[position + 1].clone(), - None => "8899".to_string(), - }; - - let rpc_url = format!("http://{}:{}", bind_address, port); - let client = RpcClient::new(rpc_url); + let client = RpcClient::new(test_validator_rpc_url(flags)); let mut count = 0; let ms_wait = 5000; while count < ms_wait { @@ -1546,7 +1536,23 @@ fn start_test_validator(cfg: &Config, flags: Option>) -> Result<(Chi println!("Unable to start test validator."); std::process::exit(1); } - Ok((validator_handle, rpc_url)) + Ok(validator_handle) +} + +// Return the URL that a validator should be running on given the config +// and flags +fn test_validator_rpc_url(flags: Option>) -> String { + let bind_address = match flags.iter().position(|f| *f == "--bind-address") { + Some(position) => flags[position + 1].clone(), + None => "127.0.0.1".to_string(), + }; + // If a custom port is defined use that for validator ready check. + let port = match flags.iter().position(|f| *f == "--rpc-port") { + Some(position) => flags[position + 1].clone(), + None => "8899".to_string(), + }; + + format!("http://{}:{}", bind_address, port) } fn deploy(cfg_override: &ConfigOverride, program_str: Option) -> Result<()> { From 88d727e7f4e7aa3260d5ce854afd420fd19a5bcd Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 15:29:52 +1300 Subject: [PATCH 10/26] Remove extra file --- cli/src/lib.rs | 1 + examples/tutorial/basic-0/client.js | 27 --------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 examples/tutorial/basic-0/client.js diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 7758b94535..f5ec43dee5 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1672,6 +1672,7 @@ fn start_test_validator( // Return the URL that a validator should be running on given the config // and flags fn test_validator_rpc_url(flags: Option>) -> String { + let flags = flags.unwrap_or_default(); let bind_address = match flags.iter().position(|f| *f == "--bind-address") { Some(position) => flags[position + 1].clone(), None => "127.0.0.1".to_string(), diff --git a/examples/tutorial/basic-0/client.js b/examples/tutorial/basic-0/client.js deleted file mode 100644 index 59b6eabd30..0000000000 --- a/examples/tutorial/basic-0/client.js +++ /dev/null @@ -1,27 +0,0 @@ -// client.js is used to introduce the reader to generating clients from IDLs. -// It is not expected users directly test with this example. For a more -// ergonomic example, see `tests/basic-0.js` in this workspace. - -const anchor = require('@project-serum/anchor'); - -// Configure the local cluster. -anchor.setProvider(anchor.Provider.local()); - -async function main() { - // #region main - // Read the generated IDL. - const idl = JSON.parse(require('fs').readFileSync('./target/idl/basic_0.json', 'utf8')); - - // Address of the deployed program. - const programId = new anchor.web3.PublicKey(''); - - // Generate the program client from IDL. - const program = new anchor.Program(idl, programId); - - // Execute the RPC. - await program.rpc.initialize(); - // #endregion main -} - -console.log('Running client.'); -main().then(() => console.log('Success')); From 0bd4341665d8f9855ab955182c0debe6495adcdf Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 15:30:57 +1300 Subject: [PATCH 11/26] Handle custom URL properly --- cli/src/config.rs | 7 +++--- cli/src/lib.rs | 57 ++++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 0b081289f0..8915a4bedb 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -457,8 +457,9 @@ fn deser_programs( .collect::>>>() } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Test { + #[serde(default, skip_serializing_if = "Option::is_none")] pub genesis: Option>, pub validator: Option, } @@ -495,8 +496,8 @@ pub struct Validator { #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, // Use DIR as ledger location - #[serde(skip_serializing_if = "Option::is_none")] - pub ledger: Option, + // #[serde(skip_serializing_if = "Option::is_none")] + // pub ledger: Option, // Keep this amount of shreds in root slots. [default: 10000] #[serde(skip_serializing_if = "Option::is_none")] pub limit_ledger_size: Option, diff --git a/cli/src/lib.rs b/cli/src/lib.rs index f5ec43dee5..aa5f0403a0 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1429,16 +1429,19 @@ fn test( deploy(cfg_override, None)?; } // Start local test validator, if needed. - let mut validator_handle = None; - let mut provider_url = cfg.provider.cluster.url(); - if is_localnet && (!skip_local_validator) { - let flags = match skip_deploy { - true => None, - false => Some(validator_flags(cfg)?), - }; - validator_handle = Some(start_test_validator(cfg, flags, true)?); - provider_url = test_validator_rpc_url(flags).as_str(); - } + let (provider_url, validator_handle) = match is_localnet && !skip_local_validator { + true => { + let flags = match skip_deploy { + true => None, + false => Some(validator_flags(cfg)?), + }; + ( + test_validator_rpc_url(&flags), + Some(start_test_validator(cfg, flags, true)?), + ) + } + false => (cfg.provider.cluster.url().to_string(), None), + }; // Setup log reader. let log_streams = stream_logs(cfg); @@ -1456,6 +1459,8 @@ fn test( .collect(); let program = args.remove(0); + println!("Anchor provider URL: {}", provider_url); + std::process::Command::new(program) .env("ANCHOR_PROVIDER_URL", provider_url) .env("ANCHOR_WALLET", cfg.provider.wallet.to_string()) @@ -1610,19 +1615,18 @@ fn start_test_validator( flags: Option>, test_log_stdout: bool, ) -> Result { - let flags = flags.unwrap_or_default(); - let test_ledger_directory: &str = match flags.iter().position(|f| *f == "--ledger") { - Some(position) => flags[position + 1].as_ref(), - None => ".anchor/test-ledger".as_ref(), - }; - let test_ledger_log_filename = format!("{}/test-ledger-log.txt", test_ledger_directory); - if Path::new(test_ledger_directory).exists() { - std::fs::remove_dir_all(test_ledger_directory)?; + // TODO respect the --ledger flag with respect to the ledger directory and + // log path? + fs::create_dir_all(".anchor")?; + let test_ledger_filename = ".anchor/test-ledger"; + let test_ledger_log_filename = ".anchor/test-ledger-log.txt"; + + if Path::new(test_ledger_filename).exists() { + std::fs::remove_dir_all(test_ledger_filename)?; } - if Path::new(&test_ledger_log_filename).exists() { + if Path::new(test_ledger_log_filename).exists() { std::fs::remove_file(test_ledger_log_filename)?; } - fs::create_dir_all(test_ledger_directory)?; // Start a validator for testing. let (test_validator_stdout, test_validator_stderr) = match test_log_stdout { @@ -1636,18 +1640,21 @@ fn start_test_validator( } false => (Stdio::inherit(), Stdio::inherit()), }; + + let rpc_url = test_validator_rpc_url(&flags); + let mut validator_handle = std::process::Command::new("solana-test-validator") .arg("--ledger") - .arg(test_ledger_directory) + .arg(test_ledger_filename) .arg("--mint") .arg(cfg.wallet_kp()?.pubkey().to_string()) - .args(&flags) + .args(flags.unwrap_or_default()) .stdout(test_validator_stdout) .stderr(test_validator_stderr) .spawn() .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; - let client = RpcClient::new(test_validator_rpc_url(Some(flags))); + let client = RpcClient::new(rpc_url); let mut count = 0; let ms_wait = 5000; while count < ms_wait { @@ -1671,8 +1678,8 @@ fn start_test_validator( // Return the URL that a validator should be running on given the config // and flags -fn test_validator_rpc_url(flags: Option>) -> String { - let flags = flags.unwrap_or_default(); +fn test_validator_rpc_url(flags: &Option>) -> String { + let flags = flags.as_ref().unwrap(); let bind_address = match flags.iter().position(|f| *f == "--bind-address") { Some(position) => flags[position + 1].clone(), None => "127.0.0.1".to_string(), From 405a4cf791ee00a1fe741be544ec7625e378c941 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 15:38:03 +1300 Subject: [PATCH 12/26] Revert "Remove extra file" This reverts commit 88d727e7f4e7aa3260d5ce854afd420fd19a5bcd. --- examples/tutorial/basic-0/client.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 examples/tutorial/basic-0/client.js diff --git a/examples/tutorial/basic-0/client.js b/examples/tutorial/basic-0/client.js new file mode 100644 index 0000000000..59b6eabd30 --- /dev/null +++ b/examples/tutorial/basic-0/client.js @@ -0,0 +1,27 @@ +// client.js is used to introduce the reader to generating clients from IDLs. +// It is not expected users directly test with this example. For a more +// ergonomic example, see `tests/basic-0.js` in this workspace. + +const anchor = require('@project-serum/anchor'); + +// Configure the local cluster. +anchor.setProvider(anchor.Provider.local()); + +async function main() { + // #region main + // Read the generated IDL. + const idl = JSON.parse(require('fs').readFileSync('./target/idl/basic_0.json', 'utf8')); + + // Address of the deployed program. + const programId = new anchor.web3.PublicKey(''); + + // Generate the program client from IDL. + const program = new anchor.Program(idl, programId); + + // Execute the RPC. + await program.rpc.initialize(); + // #endregion main +} + +console.log('Running client.'); +main().then(() => console.log('Success')); From 62163ee42d44233f70d0d4bcd3bcc7f91ca7e1a4 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 15:52:08 +1300 Subject: [PATCH 13/26] Remove unnecessary code --- cli/src/config.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 8915a4bedb..5a9feef5a8 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -457,9 +457,8 @@ fn deser_programs( .collect::>>>() } -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Test { - #[serde(default, skip_serializing_if = "Option::is_none")] pub genesis: Option>, pub validator: Option, } From 3e963aeb8ba32d3247d35cecfca940fbf1623d2f Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 15:52:21 +1300 Subject: [PATCH 14/26] Fix missing args --- cli/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index aa5f0403a0..ac81e706e4 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1462,6 +1462,7 @@ fn test( println!("Anchor provider URL: {}", provider_url); std::process::Command::new(program) + .args(args) .env("ANCHOR_PROVIDER_URL", provider_url) .env("ANCHOR_WALLET", cfg.provider.wallet.to_string()) .stdout(Stdio::inherit()) From 41a8bf48f1a7599ad4d4540a97ec7fd450813f64 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 16:03:23 +1300 Subject: [PATCH 15/26] Handle log stream for custom URL --- cli/src/lib.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index ac81e706e4..cb3ef032b9 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1429,7 +1429,7 @@ fn test( deploy(cfg_override, None)?; } // Start local test validator, if needed. - let (provider_url, validator_handle) = match is_localnet && !skip_local_validator { + let (rpc_url, validator_handle) = match is_localnet && !skip_local_validator { true => { let flags = match skip_deploy { true => None, @@ -1444,7 +1444,7 @@ fn test( }; // Setup log reader. - let log_streams = stream_logs(cfg); + let log_streams = stream_logs(cfg, &rpc_url); // Run the tests. let test_result: Result<_> = { @@ -1459,11 +1459,9 @@ fn test( .collect(); let program = args.remove(0); - println!("Anchor provider URL: {}", provider_url); - std::process::Command::new(program) .args(args) - .env("ANCHOR_PROVIDER_URL", provider_url) + .env("ANCHOR_PROVIDER_URL", rpc_url) .env("ANCHOR_WALLET", cfg.provider.wallet.to_string()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -1555,7 +1553,7 @@ fn validator_flags(cfg: &WithPath) -> Result> { Ok(flags) } -fn stream_logs(config: &WithPath) -> Result> { +fn stream_logs(config: &WithPath, rpc_url: &String) -> Result> { let program_logs_dir = ".anchor/program-logs"; if Path::new(program_logs_dir).exists() { std::fs::remove_dir_all(program_logs_dir)?; @@ -1581,7 +1579,7 @@ fn stream_logs(config: &WithPath) -> Result> { .arg("logs") .arg(metadata.address) .arg("--url") - .arg(config.provider.cluster.url()) + .arg(rpc_url) .stdout(stdio) .spawn()?; handles.push(child); @@ -2359,13 +2357,15 @@ fn localnet( )?; } - // Setup log reader. - let log_streams = stream_logs(cfg); - let flags = match skip_deploy { true => None, false => Some(validator_flags(cfg)?), }; + + // Setup log reader. + let rpc_url = test_validator_rpc_url(&flags); + let log_streams = stream_logs(cfg, &rpc_url); + let validator_handle = &mut start_test_validator(cfg, flags, false)?; std::io::stdin().lock().lines().next().unwrap().unwrap(); From 9cc060efa9945bf13e43775d2c8fdd6d0914aa3d Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 18:42:03 +1300 Subject: [PATCH 16/26] Ledger arg handling --- cli/src/config.rs | 13 ++++++++++--- cli/src/lib.rs | 42 ++++++++++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 5a9feef5a8..4884260479 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -471,8 +471,11 @@ pub struct GenesisEntry { pub program: String, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Validator { + // Clone + #[serde(skip_serializing_if = "Option::is_none")] + pub clone: Option>, // IP address to bind the validator ports. [default: 0.0.0.0] #[serde(skip_serializing_if = "Option::is_none")] pub bind_address: Option, @@ -495,8 +498,8 @@ pub struct Validator { #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, // Use DIR as ledger location - // #[serde(skip_serializing_if = "Option::is_none")] - // pub ledger: Option, + #[serde(default = "default_ledger_path")] + pub ledger: String, // Keep this amount of shreds in root slots. [default: 10000] #[serde(skip_serializing_if = "Option::is_none")] pub limit_ledger_size: Option, @@ -511,6 +514,10 @@ pub struct Validator { pub warp_slot: Option, } +fn default_ledger_path() -> String { + ".anchor/test-ledger".to_string() +} + #[derive(Debug, Clone)] pub struct Program { pub lib_name: String, diff --git a/cli/src/lib.rs b/cli/src/lib.rs index cb3ef032b9..1174999d26 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1546,7 +1546,12 @@ fn validator_flags(cfg: &WithPath) -> Result> { } for (key, value) in serde_json::to_value(&test.validator)?.as_object().unwrap() { flags.push(format!("--{}", key.replace("_", "-"))); - flags.push(value.to_string()); + let flag_value = match value.clone() { + e @ serde_json::Value::Number(_) | e @ serde_json::Value::Bool(_) => e.to_string(), + serde_json::Value::String(s) => s, + _ => "".to_string(), + }; + flags.push(flag_value); } } @@ -1614,23 +1619,23 @@ fn start_test_validator( flags: Option>, test_log_stdout: bool, ) -> Result { - // TODO respect the --ledger flag with respect to the ledger directory and - // log path? - fs::create_dir_all(".anchor")?; - let test_ledger_filename = ".anchor/test-ledger"; - let test_ledger_log_filename = ".anchor/test-ledger-log.txt"; - - if Path::new(test_ledger_filename).exists() { - std::fs::remove_dir_all(test_ledger_filename)?; + // + let (test_ledger_directory, test_ledger_log_filename) = test_validator_file_paths(&flags); + + if Path::new(&test_ledger_directory).exists() { + std::fs::remove_dir_all(&test_ledger_directory)?; } - if Path::new(test_ledger_log_filename).exists() { - std::fs::remove_file(test_ledger_log_filename)?; + + fs::create_dir_all(&test_ledger_directory)?; + + if Path::new(&test_ledger_log_filename).exists() { + std::fs::remove_file(&test_ledger_log_filename)?; } // Start a validator for testing. let (test_validator_stdout, test_validator_stderr) = match test_log_stdout { true => { - let test_validator_stdout_file = File::create(test_ledger_log_filename)?; + let test_validator_stdout_file = File::create(&test_ledger_log_filename)?; let test_validator_sterr_file = test_validator_stdout_file.try_clone()?; ( Stdio::from(test_validator_stdout_file), @@ -1643,8 +1648,6 @@ fn start_test_validator( let rpc_url = test_validator_rpc_url(&flags); let mut validator_handle = std::process::Command::new("solana-test-validator") - .arg("--ledger") - .arg(test_ledger_filename) .arg("--mint") .arg(cfg.wallet_kp()?.pubkey().to_string()) .args(flags.unwrap_or_default()) @@ -1692,6 +1695,17 @@ fn test_validator_rpc_url(flags: &Option>) -> String { format!("http://{}:{}", bind_address, port) } +fn test_validator_file_paths(flags: &Option>) -> (String, String) { + let flags = flags.as_ref().unwrap(); + let test_ledger_directory = match flags.iter().position(|f| *f == "--ledger") { + Some(position) => flags[position + 1].to_string(), + None => ".anchor/test-ledger".to_string(), + }; + let test_ledger_log_filename = format!("{}/test-ledger-log.txt", test_ledger_directory); + + (test_ledger_directory, test_ledger_log_filename) +} + fn deploy(cfg_override: &ConfigOverride, program_str: Option) -> Result<()> { with_workspace(cfg_override, |cfg| { let url = cfg.provider.cluster.url().to_string(); From ce414bf2c43c848ce66a606ab192ab60ab117940 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 21:49:13 +1300 Subject: [PATCH 17/26] Clone support --- cli/src/config.rs | 10 +++++++--- cli/src/lib.rs | 32 ++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 4884260479..abfd38a7b1 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -460,6 +460,7 @@ fn deser_programs( #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Test { pub genesis: Option>, + pub clone: Option>, pub validator: Option, } @@ -471,11 +472,14 @@ pub struct GenesisEntry { pub program: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CloneEntry { + // Base58 pubkey string. + pub address: String, +} + #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Validator { - // Clone - #[serde(skip_serializing_if = "Option::is_none")] - pub clone: Option>, // IP address to bind the validator ports. [default: 0.0.0.0] #[serde(skip_serializing_if = "Option::is_none")] pub bind_address: Option, diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 1174999d26..ee16cb113d 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1544,14 +1544,19 @@ fn validator_flags(cfg: &WithPath) -> Result> { flags.push(entry.program.clone()); } } + if let Some(clone) = &test.clone { + for entry in clone { + flags.push("--clone".to_string()); + flags.push(entry.address.clone()); + } + } for (key, value) in serde_json::to_value(&test.validator)?.as_object().unwrap() { flags.push(format!("--{}", key.replace("_", "-"))); - let flag_value = match value.clone() { - e @ serde_json::Value::Number(_) | e @ serde_json::Value::Bool(_) => e.to_string(), - serde_json::Value::String(s) => s, - _ => "".to_string(), - }; - flags.push(flag_value); + if let serde_json::Value::String(v) = value { + flags.push(v.to_string()); + } else { + flags.push(value.to_string()); + } } } @@ -1701,6 +1706,17 @@ fn test_validator_file_paths(flags: &Option>) -> (String, String) { Some(position) => flags[position + 1].to_string(), None => ".anchor/test-ledger".to_string(), }; + + if !Path::new(&test_ledger_directory).is_relative() { + // Prevent absolute paths to avoid someone using / or similar, as the + // directory gets removed + eprintln!( + "Ledger directory {} must be relative", + test_ledger_directory + ); + std::process::exit(1); + } + let test_ledger_log_filename = format!("{}/test-ledger-log.txt", test_ledger_directory); (test_ledger_directory, test_ledger_log_filename) @@ -1987,8 +2003,8 @@ fn migrate(cfg_override: &ConfigOverride) -> Result<()> { fn set_workspace_dir_or_exit() { let d = match Config::discover(&ConfigOverride::default()) { - Err(_) => { - println!("Not in anchor workspace."); + Err(err) => { + println!("Workspace configuration error: {}", err); std::process::exit(1); } Ok(d) => d, From e030c121a33450e4aa1e0261fe41f2fdcf0864e7 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Mon, 4 Oct 2021 22:03:48 +1300 Subject: [PATCH 18/26] Change URL in arg --- cli/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index ee16cb113d..c717b5695f 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1603,7 +1603,7 @@ fn stream_logs(config: &WithPath, rpc_url: &String) -> Result Date: Tue, 5 Oct 2021 11:05:38 +1300 Subject: [PATCH 19/26] Better config and URL handling --- cli/src/config.rs | 18 ++++-- cli/src/lib.rs | 147 +++++++++++++++++++++++----------------------- 2 files changed, 87 insertions(+), 78 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index abfd38a7b1..63250a4bd8 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -461,7 +461,7 @@ fn deser_programs( pub struct Test { pub genesis: Option>, pub clone: Option>, - pub validator: Option, + pub validator: Validator, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -481,8 +481,8 @@ pub struct CloneEntry { #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Validator { // IP address to bind the validator ports. [default: 0.0.0.0] - #[serde(skip_serializing_if = "Option::is_none")] - pub bind_address: Option, + #[serde(default = "default_bind_address")] + pub bind_address: String, // Range to use for dynamically assigned ports. [default: 1024-65535] #[serde(skip_serializing_if = "Option::is_none")] pub dynamic_port_range: Option, @@ -508,8 +508,8 @@ pub struct Validator { #[serde(skip_serializing_if = "Option::is_none")] pub limit_ledger_size: Option, // Enable JSON RPC on this port, and the next port for the RPC websocket. [default: 8899] - #[serde(skip_serializing_if = "Option::is_none")] - pub rpc_port: Option, + #[serde(default = "default_rpc_port")] + pub rpc_port: u16, // Override the number of slots in an epoch. #[serde(skip_serializing_if = "Option::is_none")] pub slots_per_epoch: Option, @@ -522,6 +522,14 @@ fn default_ledger_path() -> String { ".anchor/test-ledger".to_string() } +fn default_bind_address() -> String { + "0.0.0.0".to_string() +} + +fn default_rpc_port() -> u16 { + 8899 +} + #[derive(Debug, Clone)] pub struct Program { pub lib_name: String, diff --git a/cli/src/lib.rs b/cli/src/lib.rs index c717b5695f..eda3c2a3e0 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -927,7 +927,8 @@ fn verify( .join("target/verifiable/") .join(format!("{}.so", binary_name)); - let bin_ver = verify_bin(program_id, &bin_path, cfg.provider.cluster.url())?; + let url = cluster_url(&cfg); + let bin_ver = verify_bin(program_id, &bin_path, &url)?; if !bin_ver.is_verified { println!("Error: Binaries don't match"); std::process::exit(1); @@ -1052,7 +1053,8 @@ pub enum BinVerificationState { // Fetches an IDL for the given program_id. fn fetch_idl(cfg_override: &ConfigOverride, idl_addr: Pubkey) -> Result { let cfg = Config::discover(cfg_override)?.expect("Inside a workspace"); - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); let mut account = client .get_account_with_commitment(&idl_addr, CommitmentConfig::processed())? @@ -1149,7 +1151,8 @@ fn idl_set_buffer(cfg_override: &ConfigOverride, program_id: Pubkey, buffer: Pub with_workspace(cfg_override, |cfg| { let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); // Instruction to set the buffer onto the IdlAccount. let set_buffer_ix = { @@ -1201,7 +1204,8 @@ fn idl_upgrade( fn idl_authority(cfg_override: &ConfigOverride, program_id: Pubkey) -> Result<()> { with_workspace(cfg_override, |cfg| { - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); let idl_address = { let account = client .get_account_with_commitment(&program_id, CommitmentConfig::processed())? @@ -1238,7 +1242,8 @@ fn idl_set_authority( }; let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); // Instruction data. let data = @@ -1308,7 +1313,8 @@ fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl, idl_address: Pubkey) // Misc. let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); // Serialize and compress the idl. let idl_data = { @@ -1429,22 +1435,19 @@ fn test( deploy(cfg_override, None)?; } // Start local test validator, if needed. - let (rpc_url, validator_handle) = match is_localnet && !skip_local_validator { - true => { - let flags = match skip_deploy { - true => None, - false => Some(validator_flags(cfg)?), - }; - ( - test_validator_rpc_url(&flags), - Some(start_test_validator(cfg, flags, true)?), - ) - } - false => (cfg.provider.cluster.url().to_string(), None), - }; + let mut validator_handle = None; + if is_localnet && (!skip_local_validator) { + let flags = match skip_deploy { + true => None, + false => Some(validator_flags(cfg)?), + }; + validator_handle = Some(start_test_validator(cfg, flags, true)?); + } + + let url = cluster_url(&cfg); // Setup log reader. - let log_streams = stream_logs(cfg, &rpc_url); + let log_streams = stream_logs(cfg, &url); // Run the tests. let test_result: Result<_> = { @@ -1461,7 +1464,7 @@ fn test( std::process::Command::new(program) .args(args) - .env("ANCHOR_PROVIDER_URL", rpc_url) + .env("ANCHOR_PROVIDER_URL", url) .env("ANCHOR_WALLET", cfg.provider.wallet.to_string()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -1625,17 +1628,7 @@ fn start_test_validator( test_log_stdout: bool, ) -> Result { // - let (test_ledger_directory, test_ledger_log_filename) = test_validator_file_paths(&flags); - - if Path::new(&test_ledger_directory).exists() { - std::fs::remove_dir_all(&test_ledger_directory)?; - } - - fs::create_dir_all(&test_ledger_directory)?; - - if Path::new(&test_ledger_log_filename).exists() { - std::fs::remove_file(&test_ledger_log_filename)?; - } + let test_ledger_log_filename = test_validator_log_path(&cfg); // Start a validator for testing. let (test_validator_stdout, test_validator_stderr) = match test_log_stdout { @@ -1650,7 +1643,7 @@ fn start_test_validator( false => (Stdio::inherit(), Stdio::inherit()), }; - let rpc_url = test_validator_rpc_url(&flags); + let rpc_url = test_validator_rpc_url(&cfg); let mut validator_handle = std::process::Command::new("solana-test-validator") .arg("--mint") @@ -1683,48 +1676,55 @@ fn start_test_validator( Ok(validator_handle) } -// Return the URL that a validator should be running on given the config -// and flags -fn test_validator_rpc_url(flags: &Option>) -> String { - let flags = flags.as_ref().unwrap(); - let bind_address = match flags.iter().position(|f| *f == "--bind-address") { - Some(position) => flags[position + 1].clone(), - None => "127.0.0.1".to_string(), - }; - // If a custom port is defined use that for validator ready check. - let port = match flags.iter().position(|f| *f == "--rpc-port") { - Some(position) => flags[position + 1].clone(), - None => "8899".to_string(), - }; - - format!("http://{}:{}", bind_address, port) +// Return the URL that solana-test-validator should be running on given the +// configuration +fn test_validator_rpc_url(cfg: &Config) -> String { + if let Some(test) = cfg.test.as_ref() { + format!( + "http://{}:{}", + test.validator.bind_address, test.validator.rpc_port + ) + } else { + "http://localhost:8899".to_string() + } } -fn test_validator_file_paths(flags: &Option>) -> (String, String) { - let flags = flags.as_ref().unwrap(); - let test_ledger_directory = match flags.iter().position(|f| *f == "--ledger") { - Some(position) => flags[position + 1].to_string(), - None => ".anchor/test-ledger".to_string(), +// Setup and return paths to the solana-test-validator ledger directory and log +// files given the configuration +fn test_validator_log_path(cfg: &Config) -> String { + let ledger_directory = match cfg.test.as_ref() { + Some(test) => &test.validator.ledger, + None => ".anchor/test-ledger", }; - if !Path::new(&test_ledger_directory).is_relative() { + if !Path::new(&ledger_directory).is_relative() { // Prevent absolute paths to avoid someone using / or similar, as the // directory gets removed - eprintln!( - "Ledger directory {} must be relative", - test_ledger_directory - ); + eprintln!("Ledger directory {} must be relative", ledger_directory); std::process::exit(1); } + if Path::new(&ledger_directory).exists() { + std::fs::remove_dir_all(&ledger_directory).unwrap(); + } + + fs::create_dir_all(&ledger_directory).unwrap(); - let test_ledger_log_filename = format!("{}/test-ledger-log.txt", test_ledger_directory); + format!("{}/test-ledger-log.txt", ledger_directory) +} - (test_ledger_directory, test_ledger_log_filename) +fn cluster_url(cfg: &Config) -> String { + let is_localnet = cfg.provider.cluster == Cluster::Localnet; + match is_localnet { + // Cluster is Localnet, assume the intent is to use the configuration + // for solana-test-validator + true => test_validator_rpc_url(&cfg), + false => cfg.provider.cluster.url().to_string(), + } } fn deploy(cfg_override: &ConfigOverride, program_str: Option) -> Result<()> { with_workspace(cfg_override, |cfg| { - let url = cfg.provider.cluster.url().to_string(); + let url = cluster_url(&cfg); let keypair = cfg.provider.wallet.to_string(); // Deploy the programs. @@ -1798,11 +1798,12 @@ fn upgrade( let program_filepath = path.canonicalize()?.display().to_string(); with_workspace(cfg_override, |cfg| { + let url = cluster_url(&cfg); let exit = std::process::Command::new("solana") .arg("program") .arg("deploy") .arg("--url") - .arg(cfg.provider.cluster.url()) + .arg(url) .arg("--keypair") .arg(&cfg.provider.wallet.to_string()) .arg("--program-id") @@ -1830,7 +1831,8 @@ fn create_idl_account( let idl_address = IdlAccount::address(program_id); let keypair = solana_sdk::signature::read_keypair_file(keypair_path) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); let idl_data = serialize_idl(idl)?; // Run `Create instruction. @@ -1883,7 +1885,8 @@ fn create_idl_buffer( ) -> Result { let keypair = solana_sdk::signature::read_keypair_file(keypair_path) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let client = RpcClient::new(cfg.provider.cluster.url().to_string()); + let url = cluster_url(&cfg); + let client = RpcClient::new(url); let buffer = Keypair::generate(&mut OsRng); @@ -1956,7 +1959,7 @@ fn migrate(cfg_override: &ConfigOverride) -> Result<()> { with_workspace(cfg_override, |cfg| { println!("Running migration deploy script"); - let url = cfg.provider.cluster.url().to_string(); + let url = cluster_url(&cfg); let cur_dir = std::env::current_dir()?; let use_ts = @@ -2110,11 +2113,8 @@ fn shell(cfg_override: &ConfigOverride) -> Result<()> { .collect::>(), } }; - let js_code = template::node_shell( - cfg.provider.cluster.url(), - &cfg.provider.wallet.to_string(), - programs, - )?; + let url = cluster_url(&cfg); + let js_code = template::node_shell(&url, &cfg.provider.wallet.to_string(), programs)?; let mut child = std::process::Command::new("node") .args(&["-e", &js_code, "-i", "--experimental-repl-await"]) .stdout(Stdio::inherit()) @@ -2132,6 +2132,7 @@ fn shell(cfg_override: &ConfigOverride) -> Result<()> { fn run(cfg_override: &ConfigOverride, script: String) -> Result<()> { with_workspace(cfg_override, |cfg| { + let url = cluster_url(&cfg); let script = cfg .scripts .get(&script) @@ -2139,7 +2140,7 @@ fn run(cfg_override: &ConfigOverride, script: String) -> Result<()> { let exit = std::process::Command::new("bash") .arg("-c") .arg(&script) - .env("ANCHOR_PROVIDER_URL", cfg.provider.cluster.url()) + .env("ANCHOR_PROVIDER_URL", url) .env("ANCHOR_WALLET", cfg.provider.wallet.to_string()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -2393,8 +2394,8 @@ fn localnet( }; // Setup log reader. - let rpc_url = test_validator_rpc_url(&flags); - let log_streams = stream_logs(cfg, &rpc_url); + let url = test_validator_rpc_url(&cfg); + let log_streams = stream_logs(cfg, &url); let validator_handle = &mut start_test_validator(cfg, flags, false)?; From bfc5b9904d033ed77e3d3f7a0ab44e3342f151d0 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 12:03:07 +1300 Subject: [PATCH 20/26] Stream logs after validator starts in localnet command --- cli/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index eda3c2a3e0..81195ac36b 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -2393,12 +2393,12 @@ fn localnet( false => Some(validator_flags(cfg)?), }; + let validator_handle = &mut start_test_validator(cfg, flags, false)?; + // Setup log reader. let url = test_validator_rpc_url(&cfg); let log_streams = stream_logs(cfg, &url); - let validator_handle = &mut start_test_validator(cfg, flags, false)?; - std::io::stdin().lock().lines().next().unwrap().unwrap(); // Check all errors and shut down. From 528827bfee6603f9b17e1f661bbab782098823e4 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 12:08:49 +1300 Subject: [PATCH 21/26] Update comment --- cli/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 81195ac36b..716a4de7eb 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1507,8 +1507,9 @@ fn test( }) } -// Returns the solana-test-validator flags to embed the workspace programs -// in the genesis block. This allows us to run tests without every deploying. +// Returns the solana-test-validator flags. This will embed the workspace +// programs in the genesis block so we don't have to deploy every time. It also +// allows control of other solana-test-validator features. fn validator_flags(cfg: &WithPath) -> Result> { let programs = cfg.programs.get(&Cluster::Localnet); From f90a34a939f3ec072fee6727de676f96496f2560 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 13:15:59 +1300 Subject: [PATCH 22/26] Make validator key optional --- cli/src/config.rs | 2 +- cli/src/lib.rs | 39 ++++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 63250a4bd8..4fced6fade 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -461,7 +461,7 @@ fn deser_programs( pub struct Test { pub genesis: Option>, pub clone: Option>, - pub validator: Validator, + pub validator: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 716a4de7eb..5a9521e8ba 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,5 +1,6 @@ use crate::config::{ - AnchorPackage, Config, ConfigOverride, Manifest, ProgramDeployment, ProgramWorkspace, WithPath, + AnchorPackage, Config, ConfigOverride, Manifest, ProgramDeployment, ProgramWorkspace, Test, + WithPath, }; use anchor_client::Cluster; use anchor_lang::idl::{IdlAccount, IdlInstruction}; @@ -1554,12 +1555,14 @@ fn validator_flags(cfg: &WithPath) -> Result> { flags.push(entry.address.clone()); } } - for (key, value) in serde_json::to_value(&test.validator)?.as_object().unwrap() { - flags.push(format!("--{}", key.replace("_", "-"))); - if let serde_json::Value::String(v) = value { - flags.push(v.to_string()); - } else { - flags.push(value.to_string()); + if let Some(validator) = &test.validator { + for (key, value) in serde_json::to_value(validator)?.as_object().unwrap() { + flags.push(format!("--{}", key.replace("_", "-"))); + if let serde_json::Value::String(v) = value { + flags.push(v.to_string()); + } else { + flags.push(value.to_string()); + } } } } @@ -1680,22 +1683,24 @@ fn start_test_validator( // Return the URL that solana-test-validator should be running on given the // configuration fn test_validator_rpc_url(cfg: &Config) -> String { - if let Some(test) = cfg.test.as_ref() { - format!( - "http://{}:{}", - test.validator.bind_address, test.validator.rpc_port - ) - } else { - "http://localhost:8899".to_string() + match &cfg.test.as_ref() { + Some(Test { + validator: Some(validator), + .. + }) => format!("http://{}:{}", validator.bind_address, validator.rpc_port), + _ => "http://localhost:8899".to_string(), } } // Setup and return paths to the solana-test-validator ledger directory and log // files given the configuration fn test_validator_log_path(cfg: &Config) -> String { - let ledger_directory = match cfg.test.as_ref() { - Some(test) => &test.validator.ledger, - None => ".anchor/test-ledger", + let ledger_directory = match &cfg.test.as_ref() { + Some(Test { + validator: Some(validator), + .. + }) => &validator.ledger, + _ => ".anchor/test-ledger", }; if !Path::new(&ledger_directory).is_relative() { From 03b938f43eab7625d8701c820d34be62764dbd04 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 13:19:17 +1300 Subject: [PATCH 23/26] Add comment back --- cli/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 5a9521e8ba..61c0386648 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1658,6 +1658,7 @@ fn start_test_validator( .spawn() .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; + // Wait for the validator to be ready. let client = RpcClient::new(rpc_url); let mut count = 0; let ms_wait = 5000; From 30fc3d915b4ba57e4b8e585a7acb67cd825c2363 Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 13:22:22 +1300 Subject: [PATCH 24/26] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd038aaf8..4a7e4b9de5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ incremented for features. ## [Unreleased] +### Features + +* cli: Add support for configuration options for `solana-test-validator` in Anchor.toml. + ## [0.17.0] - 2021-10-03 ### Features From 4933ca41c1f9f83e272afcdf9c3d7147c31eb16a Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 15:47:34 +1300 Subject: [PATCH 25/26] Clippy --- cli/src/lib.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 61c0386648..b1fd57878e 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1152,7 +1152,7 @@ fn idl_set_buffer(cfg_override: &ConfigOverride, program_id: Pubkey, buffer: Pub with_workspace(cfg_override, |cfg| { let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let client = RpcClient::new(url); // Instruction to set the buffer onto the IdlAccount. @@ -1205,7 +1205,7 @@ fn idl_upgrade( fn idl_authority(cfg_override: &ConfigOverride, program_id: Pubkey) -> Result<()> { with_workspace(cfg_override, |cfg| { - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let client = RpcClient::new(url); let idl_address = { let account = client @@ -1243,7 +1243,7 @@ fn idl_set_authority( }; let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let client = RpcClient::new(url); // Instruction data. @@ -1314,7 +1314,7 @@ fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl, idl_address: Pubkey) // Misc. let keypair = solana_sdk::signature::read_keypair_file(&cfg.provider.wallet.to_string()) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let client = RpcClient::new(url); // Serialize and compress the idl. @@ -1445,7 +1445,7 @@ fn test( validator_handle = Some(start_test_validator(cfg, flags, true)?); } - let url = cluster_url(&cfg); + let url = cluster_url(cfg); // Setup log reader. let log_streams = stream_logs(cfg, &url); @@ -1570,7 +1570,7 @@ fn validator_flags(cfg: &WithPath) -> Result> { Ok(flags) } -fn stream_logs(config: &WithPath, rpc_url: &String) -> Result> { +fn stream_logs(config: &WithPath, rpc_url: &str) -> Result> { let program_logs_dir = ".anchor/program-logs"; if Path::new(program_logs_dir).exists() { std::fs::remove_dir_all(program_logs_dir)?; @@ -1632,7 +1632,7 @@ fn start_test_validator( test_log_stdout: bool, ) -> Result { // - let test_ledger_log_filename = test_validator_log_path(&cfg); + let test_ledger_log_filename = test_validator_log_path(cfg); // Start a validator for testing. let (test_validator_stdout, test_validator_stderr) = match test_log_stdout { @@ -1647,7 +1647,7 @@ fn start_test_validator( false => (Stdio::inherit(), Stdio::inherit()), }; - let rpc_url = test_validator_rpc_url(&cfg); + let rpc_url = test_validator_rpc_url(cfg); let mut validator_handle = std::process::Command::new("solana-test-validator") .arg("--mint") @@ -1724,14 +1724,14 @@ fn cluster_url(cfg: &Config) -> String { match is_localnet { // Cluster is Localnet, assume the intent is to use the configuration // for solana-test-validator - true => test_validator_rpc_url(&cfg), + true => test_validator_rpc_url(cfg), false => cfg.provider.cluster.url().to_string(), } } fn deploy(cfg_override: &ConfigOverride, program_str: Option) -> Result<()> { with_workspace(cfg_override, |cfg| { - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let keypair = cfg.provider.wallet.to_string(); // Deploy the programs. @@ -1805,7 +1805,7 @@ fn upgrade( let program_filepath = path.canonicalize()?.display().to_string(); with_workspace(cfg_override, |cfg| { - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let exit = std::process::Command::new("solana") .arg("program") .arg("deploy") @@ -1838,7 +1838,7 @@ fn create_idl_account( let idl_address = IdlAccount::address(program_id); let keypair = solana_sdk::signature::read_keypair_file(keypair_path) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let client = RpcClient::new(url); let idl_data = serialize_idl(idl)?; @@ -1892,7 +1892,7 @@ fn create_idl_buffer( ) -> Result { let keypair = solana_sdk::signature::read_keypair_file(keypair_path) .map_err(|_| anyhow!("Unable to read keypair file"))?; - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let client = RpcClient::new(url); let buffer = Keypair::generate(&mut OsRng); @@ -1966,7 +1966,7 @@ fn migrate(cfg_override: &ConfigOverride) -> Result<()> { with_workspace(cfg_override, |cfg| { println!("Running migration deploy script"); - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let cur_dir = std::env::current_dir()?; let use_ts = @@ -2120,7 +2120,7 @@ fn shell(cfg_override: &ConfigOverride) -> Result<()> { .collect::>(), } }; - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let js_code = template::node_shell(&url, &cfg.provider.wallet.to_string(), programs)?; let mut child = std::process::Command::new("node") .args(&["-e", &js_code, "-i", "--experimental-repl-await"]) @@ -2139,7 +2139,7 @@ fn shell(cfg_override: &ConfigOverride) -> Result<()> { fn run(cfg_override: &ConfigOverride, script: String) -> Result<()> { with_workspace(cfg_override, |cfg| { - let url = cluster_url(&cfg); + let url = cluster_url(cfg); let script = cfg .scripts .get(&script) @@ -2403,7 +2403,7 @@ fn localnet( let validator_handle = &mut start_test_validator(cfg, flags, false)?; // Setup log reader. - let url = test_validator_rpc_url(&cfg); + let url = test_validator_rpc_url(cfg); let log_streams = stream_logs(cfg, &url); std::io::stdin().lock().lines().next().unwrap().unwrap(); From 399184796acc92de0c13c9d6dae46f045dca330c Mon Sep 17 00:00:00 2001 From: Tom Linton Date: Tue, 5 Oct 2021 16:05:11 +1300 Subject: [PATCH 26/26] Always provide ledger dir argument --- cli/src/lib.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index b1fd57878e..eb091e5dc8 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1557,6 +1557,9 @@ fn validator_flags(cfg: &WithPath) -> Result> { } if let Some(validator) = &test.validator { for (key, value) in serde_json::to_value(validator)?.as_object().unwrap() { + if key == "ledger" { + continue; + }; flags.push(format!("--{}", key.replace("_", "-"))); if let serde_json::Value::String(v) = value { flags.push(v.to_string()); @@ -1632,7 +1635,7 @@ fn start_test_validator( test_log_stdout: bool, ) -> Result { // - let test_ledger_log_filename = test_validator_log_path(cfg); + let (test_ledger_directory, test_ledger_log_filename) = test_validator_file_paths(cfg); // Start a validator for testing. let (test_validator_stdout, test_validator_stderr) = match test_log_stdout { @@ -1650,6 +1653,8 @@ fn start_test_validator( let rpc_url = test_validator_rpc_url(cfg); let mut validator_handle = std::process::Command::new("solana-test-validator") + .arg("--ledger") + .arg(test_ledger_directory) .arg("--mint") .arg(cfg.wallet_kp()?.pubkey().to_string()) .args(flags.unwrap_or_default()) @@ -1695,7 +1700,7 @@ fn test_validator_rpc_url(cfg: &Config) -> String { // Setup and return paths to the solana-test-validator ledger directory and log // files given the configuration -fn test_validator_log_path(cfg: &Config) -> String { +fn test_validator_file_paths(cfg: &Config) -> (String, String) { let ledger_directory = match &cfg.test.as_ref() { Some(Test { validator: Some(validator), @@ -1716,7 +1721,10 @@ fn test_validator_log_path(cfg: &Config) -> String { fs::create_dir_all(&ledger_directory).unwrap(); - format!("{}/test-ledger-log.txt", ledger_directory) + ( + ledger_directory.to_string(), + format!("{}/test-ledger-log.txt", ledger_directory), + ) } fn cluster_url(cfg: &Config) -> String {