From 1c387da8415b41549477d4e680c5fb685b687ed7 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 19 Dec 2024 16:30:16 +0100 Subject: [PATCH 01/12] XC-258: remove useless service --- src/http.rs | 11 +++++------ src/rpc_client/eth_rpc/mod.rs | 8 ++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/http.rs b/src/http.rs index c89eb7c0..68188dd6 100644 --- a/src/http.rs +++ b/src/http.rs @@ -42,20 +42,19 @@ pub async fn json_rpc_request( vec![], )), }; - http_request(rpc_method, service, request, cycles_cost).await + http_request(rpc_method, request, cycles_cost).await } pub async fn http_request( rpc_method: MetricRpcMethod, - service: ResolvedRpcService, request: CanisterHttpRequestArgument, cycles_cost: u128, ) -> RpcResult { - let api = service.api(); - let parsed_url = match url::Url::parse(&api.url) { + let url = request.url.clone(); + let parsed_url = match url::Url::parse(&url) { Ok(url) => url, Err(_) => { - return Err(ValidationError::Custom(format!("Error parsing URL: {}", api.url)).into()) + return Err(ValidationError::Custom(format!("Error parsing URL: {}", url)).into()) } }; let host = match parsed_url.host_str() { @@ -63,7 +62,7 @@ pub async fn http_request( None => { return Err(ValidationError::Custom(format!( "Error parsing hostname from URL: {}", - api.url + url )) .into()) } diff --git a/src/rpc_client/eth_rpc/mod.rs b/src/rpc_client/eth_rpc/mod.rs index 845b9a62..9080820d 100644 --- a/src/rpc_client/eth_rpc/mod.rs +++ b/src/rpc_client/eth_rpc/mod.rs @@ -221,9 +221,7 @@ where )), }; - let response = match http_request(provider, ð_method, request, effective_size_estimate) - .await - { + let response = match http_request(ð_method, request, effective_size_estimate).await { Err(RpcError::HttpOutcallError(HttpOutcallError::IcError { code, message })) if is_response_too_large(&code, &message) => { @@ -278,12 +276,10 @@ fn resolve_api(service: &RpcService) -> Result { } async fn http_request( - service: &RpcService, method: &str, request: CanisterHttpRequestArgument, effective_response_size_estimate: u64, ) -> Result { - let service = resolve_rpc_service(service.clone())?; let cycles_cost = get_http_request_cost( request .body @@ -293,7 +289,7 @@ async fn http_request( effective_response_size_estimate, ); let rpc_method = MetricRpcMethod(method.to_string()); - crate::http::http_request(rpc_method, service, request, cycles_cost).await + crate::http::http_request(rpc_method, request, cycles_cost).await } fn http_status_code(response: &HttpResponse) -> u16 { From 59ca2443241527304370708563c166958d23fad3 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 19 Dec 2024 16:49:09 +0100 Subject: [PATCH 02/12] XC-258: added override url parameter --- candid/evm_rpc.did | 6 ++++++ evm_rpc_types/src/lifecycle/mod.rs | 8 ++++++++ tests/tests.rs | 3 +-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/candid/evm_rpc.did b/candid/evm_rpc.did index 4f38aeee..8c8b9e2a 100644 --- a/candid/evm_rpc.did +++ b/candid/evm_rpc.did @@ -107,6 +107,7 @@ type InstallArgs = record { demo : opt bool; manageApiKeys : opt vec principal; logFilter : opt LogFilter; + overrideProvider : opt OverrideProvider; }; type Regex = text; type LogFilter = variant { @@ -115,6 +116,11 @@ type LogFilter = variant { ShowPattern : Regex; HidePattern : Regex; }; +// Override resolved provider. +// Useful for testing with a local Ethereum developer environment such as foundry. +type OverrideProvider = record { + overrideUrl : opt Regex +}; type JsonRpcError = record { code : int64; message : text }; type LogEntry = record { transactionHash : opt text; diff --git a/evm_rpc_types/src/lifecycle/mod.rs b/evm_rpc_types/src/lifecycle/mod.rs index 609c6776..f61a27c7 100644 --- a/evm_rpc_types/src/lifecycle/mod.rs +++ b/evm_rpc_types/src/lifecycle/mod.rs @@ -8,6 +8,8 @@ pub struct InstallArgs { pub manage_api_keys: Option>, #[serde(rename = "logFilter")] pub log_filter: Option, + #[serde(rename = "overrideProvider")] + pub override_provider: Option } #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)] @@ -18,5 +20,11 @@ pub enum LogFilter { HidePattern(RegexString), } +#[derive(Clone, Debug, Default, PartialEq, Eq, CandidType, Serialize, Deserialize)] +pub struct OverrideProvider { + #[serde(rename = "overrideUrl")] + pub override_url: Option +} + #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)] pub struct RegexString(pub String); diff --git a/tests/tests.rs b/tests/tests.rs index 51e5783f..9ee50142 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -91,9 +91,8 @@ impl Default for EvmRpcSetup { impl EvmRpcSetup { pub fn new() -> Self { Self::with_args(InstallArgs { - manage_api_keys: None, demo: Some(true), - log_filter: None, + ..Default::default() }) } From 5100a9e8db39d71922cde6bfc834ce592849ee7f Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 19 Dec 2024 17:23:43 +0100 Subject: [PATCH 03/12] XC-258: refactor test as proptests for stable memory --- src/types.rs | 64 ++++++++++++++++------------------------------ src/types/tests.rs | 40 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 42 deletions(-) create mode 100644 src/types/tests.rs diff --git a/src/types.rs b/src/types.rs index 11833fe2..baf67086 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,3 +1,6 @@ +#[cfg(test)] +mod tests; + use crate::constants::{API_KEY_MAX_SIZE, API_KEY_REPLACE_STRING, MESSAGE_FILTER_MAX_SIZE}; use crate::memory::get_api_key; use crate::util::hostname_from_url; @@ -374,6 +377,25 @@ impl Storable for LogFilter { }; } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct OverrideProvider { + pub override_url: Option, +} + +impl Storable for OverrideProvider { + fn to_bytes(&self) -> Cow<[u8]> { + serde_json::to_vec(self) + .expect("Error while serializing `OverrideProvider`") + .into() + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + serde_json::from_slice(&bytes).expect("Error while deserializing `Storable`") + } + + const BOUND: Bound = Bound::Unbounded; +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum RpcAuth { /// API key will be used in an Authorization header as Bearer token, e.g., @@ -385,45 +407,3 @@ pub enum RpcAuth { url_pattern: &'static str, }, } - -#[cfg(test)] -mod test { - use super::{LogFilter, RegexString}; - use ic_stable_structures::Storable; - - #[test] - fn test_message_filter_storable() { - let patterns: &[RegexString] = - &["[.]", "^DEBUG ", "(.*)?", "\\?"].map(|regex| regex.into()); - let cases = [ - vec![ - (LogFilter::ShowAll, r#""ShowAll""#.to_string()), - (LogFilter::HideAll, r#""HideAll""#.to_string()), - ], - patterns - .iter() - .map(|regex| { - ( - LogFilter::ShowPattern(regex.clone()), - format!(r#"{{"ShowPattern":{:?}}}"#, regex.0), - ) - }) - .collect(), - patterns - .iter() - .map(|regex| { - ( - LogFilter::HidePattern(regex.clone()), - format!(r#"{{"HidePattern":{:?}}}"#, regex.0), - ) - }) - .collect(), - ] - .concat(); - for (filter, expected_json) in cases { - let bytes = filter.to_bytes(); - assert_eq!(String::from_utf8(bytes.to_vec()).unwrap(), expected_json); - assert_eq!(filter, LogFilter::from_bytes(bytes)); - } - } -} diff --git a/src/types/tests.rs b/src/types/tests.rs new file mode 100644 index 00000000..00f2e61e --- /dev/null +++ b/src/types/tests.rs @@ -0,0 +1,40 @@ +use super::{LogFilter, OverrideProvider, RegexString}; +use ic_stable_structures::Storable; +use proptest::prelude::{Just, Strategy}; +use proptest::{option, prop_oneof, proptest}; +use std::fmt::Debug; + +proptest! { + #[test] + fn should_encode_decode_log_filter(value in arb_log_filter()) { + test_encoding_decoding_roundtrip(&value); + } + + #[test] + fn should_encode_decode_override_provider(value in arb_override_provider()) { + test_encoding_decoding_roundtrip(&value); + } +} + +fn arb_regex() -> impl Strategy { + ".*".prop_map(|r| RegexString::from(r.as_str())) +} + +fn arb_log_filter() -> impl Strategy { + prop_oneof![ + Just(LogFilter::ShowAll), + Just(LogFilter::HideAll), + arb_regex().prop_map(LogFilter::ShowPattern), + arb_regex().prop_map(LogFilter::HidePattern), + ] +} + +fn arb_override_provider() -> impl Strategy { + option::of(arb_regex()).prop_map(|override_url| OverrideProvider { override_url }) +} + +fn test_encoding_decoding_roundtrip(value: &T) { + let bytes = value.to_bytes(); + let decoded_value = T::from_bytes(bytes); + assert_eq!(value, &decoded_value); +} From 89b12defcc97ec6b542bea40241e92e6d81cfb5e Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 19 Dec 2024 17:41:30 +0100 Subject: [PATCH 04/12] XC-258: store in stable memory --- evm_rpc_types/src/lib.rs | 2 +- src/main.rs | 10 +++++----- src/memory.rs | 19 +++++++++++++++++-- src/types.rs | 10 ++++++++++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/evm_rpc_types/src/lib.rs b/evm_rpc_types/src/lib.rs index 93324464..b8eba599 100644 --- a/evm_rpc_types/src/lib.rs +++ b/evm_rpc_types/src/lib.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; -pub use lifecycle::{InstallArgs, LogFilter, RegexString}; +pub use lifecycle::{InstallArgs, LogFilter, OverrideProvider, RegexString}; pub use request::{ AccessList, AccessListEntry, BlockTag, CallArgs, FeeHistoryArgs, GetLogsArgs, GetTransactionCountArgs, TransactionRequest, diff --git a/src/main.rs b/src/main.rs index 0da4d43e..7237c690 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,13 +4,10 @@ use evm_rpc::candid_rpc::CandidRpcClient; use evm_rpc::constants::NODES_IN_SUBNET; use evm_rpc::http::get_http_response_body; use evm_rpc::logs::INFO; -use evm_rpc::memory::{ - insert_api_key, is_api_key_principal, is_demo_active, remove_api_key, set_api_key_principals, - set_demo_active, set_log_filter, -}; +use evm_rpc::memory::{insert_api_key, is_api_key_principal, is_demo_active, remove_api_key, set_api_key_principals, set_demo_active, set_log_filter, set_override_provider}; use evm_rpc::metrics::encode_metrics; use evm_rpc::providers::{find_provider, resolve_rpc_service, PROVIDERS, SERVICE_PROVIDER_MAP}; -use evm_rpc::types::{LogFilter, Provider, ProviderId, RpcAccess, RpcAuth}; +use evm_rpc::types::{LogFilter, OverrideProvider, Provider, ProviderId, RpcAccess, RpcAuth}; use evm_rpc::{ http::{json_rpc_request, transform_http_request}, http_types, @@ -271,6 +268,9 @@ fn post_upgrade(args: evm_rpc_types::InstallArgs) { if let Some(filter) = args.log_filter { set_log_filter(LogFilter::from(filter)) } + if let Some(override_provider) =args.override_provider { + set_override_provider(OverrideProvider::from(override_provider)); + } } #[query(hidden = true)] diff --git a/src/memory.rs b/src/memory.rs index 03ede56c..9468f19f 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -7,12 +7,13 @@ use ic_stable_structures::{ use ic_stable_structures::{Cell, StableBTreeMap}; use std::cell::RefCell; -use crate::types::{ApiKey, LogFilter, Metrics, ProviderId}; +use crate::types::{ApiKey, LogFilter, Metrics, OverrideProvider, ProviderId}; const IS_DEMO_ACTIVE_MEMORY_ID: MemoryId = MemoryId::new(4); const API_KEY_MAP_MEMORY_ID: MemoryId = MemoryId::new(5); const MANAGE_API_KEYS_MEMORY_ID: MemoryId = MemoryId::new(6); const LOG_FILTER_MEMORY_ID: MemoryId = MemoryId::new(7); +const OVERRIDE_PROVIDER_MEMORY_ID: MemoryId = MemoryId::new(8); type StableMemory = VirtualMemory; @@ -31,7 +32,9 @@ thread_local! { static MANAGE_API_KEYS: RefCell> = RefCell::new(ic_stable_structures::Vec::init(MEMORY_MANAGER.with_borrow(|m| m.get(MANAGE_API_KEYS_MEMORY_ID))).expect("Unable to read API key principals from stable memory")); static LOG_FILTER: RefCell> = - RefCell::new(ic_stable_structures::Cell::init(MEMORY_MANAGER.with_borrow(|m| m.get(LOG_FILTER_MEMORY_ID)), LogFilter::default()).expect("Unable to read log message filter from stable memory")); + RefCell::new(Cell::init(MEMORY_MANAGER.with_borrow(|m| m.get(LOG_FILTER_MEMORY_ID)), LogFilter::default()).expect("Unable to read log message filter from stable memory")); + static OVERRIDE_PROVIDER: RefCell> = + RefCell::new(Cell::init(MEMORY_MANAGER.with_borrow(|m| m.get(OVERRIDE_PROVIDER_MEMORY_ID)), OverrideProvider::default()).expect("Unable to read provider override from stable memory")); } pub fn get_api_key(provider_id: ProviderId) -> Option { @@ -86,6 +89,18 @@ pub fn set_log_filter(filter: LogFilter) { }); } +pub fn get_override_provider() -> OverrideProvider { + OVERRIDE_PROVIDER.with_borrow(|provider| provider.get().clone()) +} + +pub fn set_override_provider(provider: OverrideProvider) { + OVERRIDE_PROVIDER.with_borrow_mut(|state| { + state + .set(provider) + .expect("Error while updating override provider") + }); +} + pub fn next_request_id() -> u64 { UNSTABLE_HTTP_REQUEST_COUNTER.with_borrow_mut(|counter| { let current_request_id = *counter; diff --git a/src/types.rs b/src/types.rs index baf67086..2656ebbd 100644 --- a/src/types.rs +++ b/src/types.rs @@ -382,6 +382,16 @@ pub struct OverrideProvider { pub override_url: Option, } +impl From for OverrideProvider { + fn from( + evm_rpc_types::OverrideProvider { override_url }: evm_rpc_types::OverrideProvider, + ) -> Self { + Self { + override_url: override_url.map(RegexString::from), + } + } +} + impl Storable for OverrideProvider { fn to_bytes(&self) -> Cow<[u8]> { serde_json::to_vec(self) From a815ed760838dca2caa2e2b353cf4371880a39dc Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 07:55:52 +0100 Subject: [PATCH 05/12] XC-258: clippy --- evm_rpc_types/src/lifecycle/mod.rs | 4 ++-- src/main.rs | 7 +++++-- src/types.rs | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/evm_rpc_types/src/lifecycle/mod.rs b/evm_rpc_types/src/lifecycle/mod.rs index f61a27c7..57c7e20a 100644 --- a/evm_rpc_types/src/lifecycle/mod.rs +++ b/evm_rpc_types/src/lifecycle/mod.rs @@ -9,7 +9,7 @@ pub struct InstallArgs { #[serde(rename = "logFilter")] pub log_filter: Option, #[serde(rename = "overrideProvider")] - pub override_provider: Option + pub override_provider: Option, } #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)] @@ -23,7 +23,7 @@ pub enum LogFilter { #[derive(Clone, Debug, Default, PartialEq, Eq, CandidType, Serialize, Deserialize)] pub struct OverrideProvider { #[serde(rename = "overrideUrl")] - pub override_url: Option + pub override_url: Option, } #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)] diff --git a/src/main.rs b/src/main.rs index 7237c690..f22637a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,10 @@ use evm_rpc::candid_rpc::CandidRpcClient; use evm_rpc::constants::NODES_IN_SUBNET; use evm_rpc::http::get_http_response_body; use evm_rpc::logs::INFO; -use evm_rpc::memory::{insert_api_key, is_api_key_principal, is_demo_active, remove_api_key, set_api_key_principals, set_demo_active, set_log_filter, set_override_provider}; +use evm_rpc::memory::{ + insert_api_key, is_api_key_principal, is_demo_active, remove_api_key, set_api_key_principals, + set_demo_active, set_log_filter, set_override_provider, +}; use evm_rpc::metrics::encode_metrics; use evm_rpc::providers::{find_provider, resolve_rpc_service, PROVIDERS, SERVICE_PROVIDER_MAP}; use evm_rpc::types::{LogFilter, OverrideProvider, Provider, ProviderId, RpcAccess, RpcAuth}; @@ -268,7 +271,7 @@ fn post_upgrade(args: evm_rpc_types::InstallArgs) { if let Some(filter) = args.log_filter { set_log_filter(LogFilter::from(filter)) } - if let Some(override_provider) =args.override_provider { + if let Some(override_provider) = args.override_provider { set_override_provider(OverrideProvider::from(override_provider)); } } diff --git a/src/types.rs b/src/types.rs index 2656ebbd..07060d69 100644 --- a/src/types.rs +++ b/src/types.rs @@ -6,6 +6,7 @@ use crate::memory::get_api_key; use crate::util::hostname_from_url; use crate::validate::validate_api_key; use candid::CandidType; +use evm_rpc_types::RpcApi; use ic_cdk::api::call::RejectionCode; use ic_cdk::api::management_canister::http_request::HttpHeader; use ic_stable_structures::storable::Bound; From 0d86417f31d900582c4622fea3dc563af202a10c Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 08:19:35 +0100 Subject: [PATCH 06/12] XC-258: add replacement pattern --- candid/evm_rpc.did | 6 +++++- evm_rpc_types/src/lifecycle/mod.rs | 8 +++++++- src/types.rs | 13 +++++++++++-- src/types/tests.rs | 11 +++++++++-- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/candid/evm_rpc.did b/candid/evm_rpc.did index 8c8b9e2a..d2f43fae 100644 --- a/candid/evm_rpc.did +++ b/candid/evm_rpc.did @@ -116,10 +116,14 @@ type LogFilter = variant { ShowPattern : Regex; HidePattern : Regex; }; +type RegexSubstitution = record { + pattern : Regex; + replacement: text; +}; // Override resolved provider. // Useful for testing with a local Ethereum developer environment such as foundry. type OverrideProvider = record { - overrideUrl : opt Regex + overrideUrl : opt RegexSubstitution }; type JsonRpcError = record { code : int64; message : text }; type LogEntry = record { diff --git a/evm_rpc_types/src/lifecycle/mod.rs b/evm_rpc_types/src/lifecycle/mod.rs index 57c7e20a..a7c8b66d 100644 --- a/evm_rpc_types/src/lifecycle/mod.rs +++ b/evm_rpc_types/src/lifecycle/mod.rs @@ -23,8 +23,14 @@ pub enum LogFilter { #[derive(Clone, Debug, Default, PartialEq, Eq, CandidType, Serialize, Deserialize)] pub struct OverrideProvider { #[serde(rename = "overrideUrl")] - pub override_url: Option, + pub override_url: Option, } #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)] pub struct RegexString(pub String); + +#[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)] +pub struct RegexSubstitution { + pub pattern: RegexString, + pub replacement: String, +} diff --git a/src/types.rs b/src/types.rs index 07060d69..cf89a4aa 100644 --- a/src/types.rs +++ b/src/types.rs @@ -347,6 +347,12 @@ impl RegexString { } } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RegexSubstitution { + pub pattern: RegexString, + pub replacement: String, +} + impl LogFilter { pub fn is_match(&self, message: &str) -> bool { match self { @@ -380,7 +386,7 @@ impl Storable for LogFilter { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct OverrideProvider { - pub override_url: Option, + pub override_url: Option, } impl From for OverrideProvider { @@ -388,7 +394,10 @@ impl From for OverrideProvider { evm_rpc_types::OverrideProvider { override_url }: evm_rpc_types::OverrideProvider, ) -> Self { Self { - override_url: override_url.map(RegexString::from), + override_url: override_url.map(|substitution| RegexSubstitution { + pattern: RegexString::from(substitution.pattern), + replacement: substitution.replacement, + }), } } } diff --git a/src/types/tests.rs b/src/types/tests.rs index 00f2e61e..0d3ec475 100644 --- a/src/types/tests.rs +++ b/src/types/tests.rs @@ -1,4 +1,4 @@ -use super::{LogFilter, OverrideProvider, RegexString}; +use super::{LogFilter, OverrideProvider, RegexString, RegexSubstitution}; use ic_stable_structures::Storable; use proptest::prelude::{Just, Strategy}; use proptest::{option, prop_oneof, proptest}; @@ -20,6 +20,13 @@ fn arb_regex() -> impl Strategy { ".*".prop_map(|r| RegexString::from(r.as_str())) } +fn arb_regex_substitution() -> impl Strategy { + (arb_regex(), ".*").prop_map(|(pattern, replacement)| RegexSubstitution { + pattern, + replacement, + }) +} + fn arb_log_filter() -> impl Strategy { prop_oneof![ Just(LogFilter::ShowAll), @@ -30,7 +37,7 @@ fn arb_log_filter() -> impl Strategy { } fn arb_override_provider() -> impl Strategy { - option::of(arb_regex()).prop_map(|override_url| OverrideProvider { override_url }) + option::of(arb_regex_substitution()).prop_map(|override_url| OverrideProvider { override_url }) } fn test_encoding_decoding_roundtrip(value: &T) { From 2e60d5a7bdbba2664a445fa2d65cc8f7473908bc Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 09:07:48 +0100 Subject: [PATCH 07/12] XC-258: ensure Regex is valid --- evm_rpc_types/src/lib.rs | 2 +- src/main.rs | 7 +++-- src/types.rs | 58 +++++++++++++++++++++++++++------------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/evm_rpc_types/src/lib.rs b/evm_rpc_types/src/lib.rs index b8eba599..550451c3 100644 --- a/evm_rpc_types/src/lib.rs +++ b/evm_rpc_types/src/lib.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; -pub use lifecycle::{InstallArgs, LogFilter, OverrideProvider, RegexString}; +pub use lifecycle::{InstallArgs, LogFilter, OverrideProvider, RegexString, RegexSubstitution}; pub use request::{ AccessList, AccessListEntry, BlockTag, CallArgs, FeeHistoryArgs, GetLogsArgs, GetTransactionCountArgs, TransactionRequest, diff --git a/src/main.rs b/src/main.rs index f22637a5..dae8d5ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -269,10 +269,13 @@ fn post_upgrade(args: evm_rpc_types::InstallArgs) { set_api_key_principals(principals); } if let Some(filter) = args.log_filter { - set_log_filter(LogFilter::from(filter)) + set_log_filter(LogFilter::try_from(filter).expect("ERROR: Invalid log filter")); } if let Some(override_provider) = args.override_provider { - set_override_provider(OverrideProvider::from(override_provider)); + set_override_provider( + OverrideProvider::try_from(override_provider) + .expect("ERROR: invalid override provider"), + ); } } diff --git a/src/types.rs b/src/types.rs index cf89a4aa..4660cfeb 100644 --- a/src/types.rs +++ b/src/types.rs @@ -314,23 +314,32 @@ pub enum LogFilter { HidePattern(RegexString), } -impl From for LogFilter { - fn from(value: evm_rpc_types::LogFilter) -> Self { - match value { +impl TryFrom for LogFilter { + type Error = regex::Error; + + fn try_from(value: evm_rpc_types::LogFilter) -> Result { + Ok(match value { evm_rpc_types::LogFilter::ShowAll => LogFilter::ShowAll, evm_rpc_types::LogFilter::HideAll => LogFilter::HideAll, - evm_rpc_types::LogFilter::ShowPattern(regex) => LogFilter::ShowPattern(regex.into()), - evm_rpc_types::LogFilter::HidePattern(regex) => LogFilter::HidePattern(regex.into()), - } + evm_rpc_types::LogFilter::ShowPattern(regex) => { + LogFilter::ShowPattern(RegexString::try_from(regex)?) + } + evm_rpc_types::LogFilter::HidePattern(regex) => { + LogFilter::HidePattern(RegexString::try_from(regex)?) + } + }) } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct RegexString(String); -impl From for RegexString { - fn from(value: evm_rpc_types::RegexString) -> Self { - RegexString(value.0) +impl TryFrom for RegexString { + type Error = regex::Error; + + fn try_from(value: evm_rpc_types::RegexString) -> Result { + let ensure_regex_is_valid = Regex::new(&value.0)?; + Ok(Self(ensure_regex_is_valid.as_str().to_string())) } } @@ -353,6 +362,17 @@ pub struct RegexSubstitution { pub replacement: String, } +impl TryFrom for RegexSubstitution { + type Error = regex::Error; + + fn try_from(value: evm_rpc_types::RegexSubstitution) -> Result { + Ok(Self { + pattern: RegexString::try_from(value.pattern)?, + replacement: value.replacement, + }) + } +} + impl LogFilter { pub fn is_match(&self, message: &str) -> bool { match self { @@ -389,16 +409,18 @@ pub struct OverrideProvider { pub override_url: Option, } -impl From for OverrideProvider { - fn from( +impl TryFrom for OverrideProvider { + type Error = regex::Error; + + fn try_from( evm_rpc_types::OverrideProvider { override_url }: evm_rpc_types::OverrideProvider, - ) -> Self { - Self { - override_url: override_url.map(|substitution| RegexSubstitution { - pattern: RegexString::from(substitution.pattern), - replacement: substitution.replacement, - }), - } + ) -> Result { + override_url + .map(RegexSubstitution::try_from) + .transpose() + .map(|substitution| Self { + override_url: substitution, + }) } } From 5efd5d641f55de531cc73bc0f9e4418a435f63b8 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 11:04:24 +0100 Subject: [PATCH 08/12] XC-258: unit test for applying replacement --- src/types.rs | 38 +++++++++++++++++++-- src/types/tests.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/types.rs b/src/types.rs index 4660cfeb..2f393199 100644 --- a/src/types.rs +++ b/src/types.rs @@ -350,9 +350,15 @@ impl From<&str> for RegexString { } impl RegexString { + /// Compile the string into a regular expression. + /// + /// This is a relatively expensive operation that's currently not cached. + pub fn compile(&self) -> Result { + Regex::new(&self.0) + } + pub fn try_is_valid(&self, value: &str) -> Result { - // Currently only used in the local replica. This can be optimized if eventually used in production. - Ok(Regex::new(&self.0)?.is_match(value)) + Ok(self.compile()?.is_match(value)) } } @@ -438,6 +444,34 @@ impl Storable for OverrideProvider { const BOUND: Bound = Bound::Unbounded; } +impl OverrideProvider { + /// Override the resolved provider API (url and headers). + /// + /// # Limitations + /// + /// Currently, only the url can be replaced by regular expression. Headers will be reset. + /// + /// # Security considerations + /// + /// The resolved provider API may contain sensitive data (such as API keys) that may be extracted + /// by using the override mechanism. Since only the controller of the canister can set the override parameters, + /// upon canister initialization or upgrade, it's the controller's responsibility to ensure that this is not a problem + /// (e.g., if only used for local development). + pub fn apply(&self, api: RpcApi) -> Result { + match &self.override_url { + None => Ok(api), + Some(substitution) => { + let regex = substitution.pattern.compile()?; + let new_url = regex.replace_all(&api.url, &substitution.replacement); + Ok(RpcApi { + url: new_url.to_string(), + headers: None, + }) + } + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum RpcAuth { /// API key will be used in an Authorization header as Bearer token, e.g., diff --git a/src/types/tests.rs b/src/types/tests.rs index 0d3ec475..26f74ffe 100644 --- a/src/types/tests.rs +++ b/src/types/tests.rs @@ -45,3 +45,86 @@ fn test_encoding_decoding_roundtrip(value: &T) let decoded_value = T::from_bytes(bytes); assert_eq!(value, &decoded_value); } + +mod override_provider { + use crate::providers::PROVIDERS; + use crate::types::{OverrideProvider, RegexSubstitution}; + use evm_rpc_types::RpcApi; + use ic_cdk::api::management_canister::http_request::HttpHeader; + + #[test] + fn should_override_provider_with_localhost() { + let override_provider = override_to_localhost(); + for provider in PROVIDERS { + let overriden_provider = override_provider.apply(provider.api()); + assert_eq!( + overriden_provider, + Ok(RpcApi { + url: "http://localhost:8545".to_string(), + headers: None + }) + ) + } + } + + #[test] + fn should_be_noop_when_empty() { + let no_override = OverrideProvider::default(); + for provider in PROVIDERS { + let initial_api = provider.api(); + let overriden_api = no_override.apply(initial_api.clone()); + assert_eq!(Ok(initial_api), overriden_api); + } + } + + #[test] + fn should_use_replacement_pattern() { + let identity_override = OverrideProvider { + override_url: Some(RegexSubstitution { + pattern: "(?.*)".into(), + replacement: "$url".to_string(), + }), + }; + for provider in PROVIDERS { + let initial_api = provider.api(); + let overriden_provider = identity_override.apply(initial_api.clone()); + assert_eq!(overriden_provider, Ok(initial_api)) + } + } + + #[test] + fn should_override_headers() { + let identity_override = OverrideProvider { + override_url: Some(RegexSubstitution { + pattern: "(.*)".into(), + replacement: "$1".to_string(), + }), + }; + for provider in PROVIDERS { + let provider_with_headers = RpcApi { + headers: Some(vec![HttpHeader { + name: "key".to_string(), + value: "123".to_string(), + }]), + ..provider.api() + }; + let overriden_provider = identity_override.apply(provider_with_headers.clone()); + assert_eq!( + overriden_provider, + Ok(RpcApi { + url: provider_with_headers.url, + headers: None + }) + ) + } + } + + fn override_to_localhost() -> OverrideProvider { + OverrideProvider { + override_url: Some(RegexSubstitution { + pattern: "^https://.*".into(), + replacement: "http://localhost:8545".to_string(), + }), + } + } +} From b820e7048a036c6d2754433e6ddc785d7374193e Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 11:24:41 +0100 Subject: [PATCH 09/12] XC-258: apply override --- src/http.rs | 3 +- src/rpc_client/eth_rpc/mod.rs | 15 +++++--- src/types.rs | 71 +++++++++++++++++++---------------- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/src/http.rs b/src/http.rs index 68188dd6..e1b041d5 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,3 +1,4 @@ +use crate::memory::get_override_provider; use crate::{ accounting::{get_cost_with_collateral, get_http_request_cost}, add_metric_entry, @@ -20,7 +21,7 @@ pub async fn json_rpc_request( max_response_bytes: u64, ) -> RpcResult { let cycles_cost = get_http_request_cost(json_rpc_payload.len() as u64, max_response_bytes); - let api = service.api(); + let api = service.api(&get_override_provider())?; let mut request_headers = api.headers.unwrap_or_default(); if !request_headers .iter() diff --git a/src/rpc_client/eth_rpc/mod.rs b/src/rpc_client/eth_rpc/mod.rs index 9080820d..9841e1f0 100644 --- a/src/rpc_client/eth_rpc/mod.rs +++ b/src/rpc_client/eth_rpc/mod.rs @@ -3,7 +3,7 @@ use crate::accounting::get_http_request_cost; use crate::logs::{DEBUG, TRACE_HTTP}; -use crate::memory::next_request_id; +use crate::memory::{get_override_provider, next_request_id}; use crate::providers::resolve_rpc_service; use crate::rpc_client::eth_rpc_error::{sanitize_send_raw_transaction_result, Parser}; use crate::rpc_client::json::requests::JsonRpcRequest; @@ -11,9 +11,9 @@ use crate::rpc_client::json::responses::{ Block, FeeHistory, JsonRpcReply, JsonRpcResult, LogEntry, TransactionReceipt, }; use crate::rpc_client::numeric::{TransactionCount, Wei}; -use crate::types::MetricRpcMethod; +use crate::types::{MetricRpcMethod, OverrideProvider}; use candid::candid_method; -use evm_rpc_types::{HttpOutcallError, ProviderError, RpcApi, RpcError, RpcService}; +use evm_rpc_types::{HttpOutcallError, RpcApi, RpcError, RpcService}; use ic_canister_log::log; use ic_cdk::api::call::RejectionCode; use ic_cdk::api::management_canister::http_request::{ @@ -181,7 +181,7 @@ where method: eth_method.clone(), id: 1, }; - let api = resolve_api(provider)?; + let api = resolve_api(provider, &get_override_provider())?; let url = &api.url; let mut headers = vec![HttpHeader { name: "Content-Type".to_string(), @@ -271,8 +271,11 @@ where } } -fn resolve_api(service: &RpcService) -> Result { - Ok(resolve_rpc_service(service.clone())?.api()) +fn resolve_api( + service: &RpcService, + override_provider: &OverrideProvider, +) -> Result { + resolve_rpc_service(service.clone())?.api(override_provider) } async fn http_request( diff --git a/src/types.rs b/src/types.rs index 2f393199..f71d1a3f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -6,7 +6,7 @@ use crate::memory::get_api_key; use crate::util::hostname_from_url; use crate::validate::validate_api_key; use candid::CandidType; -use evm_rpc_types::RpcApi; +use evm_rpc_types::{RpcApi, RpcError, ValidationError}; use ic_cdk::api::call::RejectionCode; use ic_cdk::api::management_canister::http_request::HttpHeader; use ic_stable_structures::storable::Bound; @@ -19,16 +19,21 @@ use std::fmt; use zeroize::{Zeroize, ZeroizeOnDrop}; pub enum ResolvedRpcService { - Api(evm_rpc_types::RpcApi), + Api(RpcApi), Provider(Provider), } impl ResolvedRpcService { - pub fn api(&self) -> evm_rpc_types::RpcApi { - match self { + pub fn api(&self, override_provider: &OverrideProvider) -> Result { + let initial_api = match self { Self::Api(api) => api.clone(), Self::Provider(provider) => provider.api(), - } + }; + override_provider.apply(initial_api).map_err(|regex_error| { + RpcError::ValidationError(ValidationError::Custom(format!( + "BUG: regex should have been validated when initially set. Error: {regex_error}" + ))) + }) } } @@ -415,6 +420,34 @@ pub struct OverrideProvider { pub override_url: Option, } +impl OverrideProvider { + /// Override the resolved provider API (url and headers). + /// + /// # Limitations + /// + /// Currently, only the url can be replaced by regular expression. Headers will be reset. + /// + /// # Security considerations + /// + /// The resolved provider API may contain sensitive data (such as API keys) that may be extracted + /// by using the override mechanism. Since only the controller of the canister can set the override parameters, + /// upon canister initialization or upgrade, it's the controller's responsibility to ensure that this is not a problem + /// (e.g., if only used for local development). + pub fn apply(&self, api: RpcApi) -> Result { + match &self.override_url { + None => Ok(api), + Some(substitution) => { + let regex = substitution.pattern.compile()?; + let new_url = regex.replace_all(&api.url, &substitution.replacement); + Ok(RpcApi { + url: new_url.to_string(), + headers: None, + }) + } + } + } +} + impl TryFrom for OverrideProvider { type Error = regex::Error; @@ -444,34 +477,6 @@ impl Storable for OverrideProvider { const BOUND: Bound = Bound::Unbounded; } -impl OverrideProvider { - /// Override the resolved provider API (url and headers). - /// - /// # Limitations - /// - /// Currently, only the url can be replaced by regular expression. Headers will be reset. - /// - /// # Security considerations - /// - /// The resolved provider API may contain sensitive data (such as API keys) that may be extracted - /// by using the override mechanism. Since only the controller of the canister can set the override parameters, - /// upon canister initialization or upgrade, it's the controller's responsibility to ensure that this is not a problem - /// (e.g., if only used for local development). - pub fn apply(&self, api: RpcApi) -> Result { - match &self.override_url { - None => Ok(api), - Some(substitution) => { - let regex = substitution.pattern.compile()?; - let new_url = regex.replace_all(&api.url, &substitution.replacement); - Ok(RpcApi { - url: new_url.to_string(), - headers: None, - }) - } - } - } -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum RpcAuth { /// API key will be used in an Authorization header as Bearer token, e.g., From 8ee74ed0ff780ba4f3876833479dd1d8a01ff797 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 13:49:57 +0100 Subject: [PATCH 10/12] XC-258: CI: local examples targeting Foundry --- .github/workflows/ci.yml | 5 ++++- dfx.json | 7 +++++++ scripts/e2e | 1 + scripts/examples | 7 ++++--- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a92d400..e6048e6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,10 @@ jobs: run: scripts/e2e - name: Run examples - run: scripts/examples + run: scripts/examples evm_rpc_local 'Number = 20000000' + + - name: Run local examples with Foundry + run: scripts/examples evm_rpc_local 'Number = 0' - name: Check formatting run: cargo fmt --all -- --check diff --git a/dfx.json b/dfx.json index 2fcb0c8d..44e6f54d 100644 --- a/dfx.json +++ b/dfx.json @@ -24,6 +24,13 @@ "gzip": true, "init_arg": "(record {demo = opt true})" }, + "evm_rpc_local": { + "candid": "candid/evm_rpc.did", + "type": "rust", + "package": "evm_rpc", + "gzip": true, + "init_arg": "( record { overrideProvider = opt record { overrideUrl = opt record { pattern = \".*\"; replacement = \"http://127.0.0.1:8545\" } } })" + }, "evm_rpc_staging": { "candid": "candid/evm_rpc.did", "type": "rust", diff --git a/scripts/e2e b/scripts/e2e index 7494c0da..34ccb8c7 100755 --- a/scripts/e2e +++ b/scripts/e2e @@ -4,6 +4,7 @@ dfx canister create --all && npm run generate && dfx deploy evm_rpc --mode reinstall -y && + dfx deploy evm_rpc_local --mode reinstall -y && dfx deploy evm_rpc_demo --mode reinstall -y && dfx deploy evm_rpc_staging --mode reinstall -y && dfx deploy e2e_rust && diff --git a/scripts/examples b/scripts/examples index ff5d1611..159874e9 100755 --- a/scripts/examples +++ b/scripts/examples @@ -1,16 +1,17 @@ #!/usr/bin/env bash # Run a variety of example RPC calls. +CANISTER_ID=${1:-evm_rpc} +# Use concrete block height to avoid flakiness on CI +BLOCK_HEIGHT=${2:-'Number = 20000000'} + NETWORK=local IDENTITY=default -CANISTER_ID=evm_rpc CYCLES=10000000000 WALLET=$(dfx identity get-wallet --network=$NETWORK --identity=$IDENTITY) RPC_SERVICE="EthMainnet=variant {PublicNode}" RPC_SERVICES=EthMainnet RPC_CONFIG="opt record {responseConsensus = opt variant {Threshold = record {total = opt (3 : nat8); min = 2 : nat8}}}" -# Use concrete block height to avoid flakiness on CI -BLOCK_HEIGHT="Number = 20000000" FLAGS="--network=$NETWORK --identity=$IDENTITY --with-cycles=$CYCLES --wallet=$WALLET" From da5c0d0c7f74e9441a310fc749796a55bc80e644 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 13:59:43 +0100 Subject: [PATCH 11/12] XC-258: install Foundry --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6048e6f..e71b21fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,8 +92,14 @@ jobs: run: scripts/e2e - name: Run examples - run: scripts/examples evm_rpc_local 'Number = 20000000' + run: scripts/examples evm_rpc 'Number = 20000000' + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + - name: Run anvil + run: anvil + - name: Run local examples with Foundry run: scripts/examples evm_rpc_local 'Number = 0' From 253552816fddd2c11c134fc946d3267802c8cfd6 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Fri, 20 Dec 2024 14:08:11 +0100 Subject: [PATCH 12/12] XC-258: anvil in background --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e71b21fb..caef2aca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,7 @@ jobs: uses: foundry-rs/foundry-toolchain@v1 - name: Run anvil - run: anvil + run: anvil & - name: Run local examples with Foundry run: scripts/examples evm_rpc_local 'Number = 0'