diff --git a/Cargo.lock b/Cargo.lock index 38e9f1f35f..70b42a9f80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3780,22 +3780,30 @@ dependencies = [ name = "mc-fog-ledger-connection" version = "4.1.0-pre0" dependencies = [ + "aes-gcm", "displaydoc", + "futures", "grpcio", "mc-api", + "mc-attest-ake", + "mc-attest-core", "mc-attest-verifier", "mc-blockchain-types", "mc-common", "mc-crypto-keys", + "mc-crypto-noise", + "mc-crypto-rand", "mc-fog-api", "mc-fog-enclave-connection", "mc-fog-types", "mc-fog-uri", "mc-transaction-core", "mc-util-grpc", + "mc-util-serial", "mc-util-uri", "protobuf", "retry", + "sha2 0.10.6", ] [[package]] diff --git a/fog/ledger/connection/Cargo.toml b/fog/ledger/connection/Cargo.toml index df3b83d5e5..feb2395071 100644 --- a/fog/ledger/connection/Cargo.toml +++ b/fog/ledger/connection/Cargo.toml @@ -10,12 +10,17 @@ rust-version = { workspace = true } [dependencies] # mobilecoin mc-api = { path = "../../../api" } +mc-attest-ake = { path = "../../../attest/ake" } +mc-attest-core = { path = "../../../attest/core" } mc-attest-verifier = { path = "../../../attest/verifier" } mc-blockchain-types = { path = "../../../blockchain/types" } mc-common = { path = "../../../common", features = ["log"] } mc-crypto-keys = { path = "../../../crypto/keys" } +mc-crypto-noise = { path = "../../../crypto/noise" } +mc-crypto-rand = { path = "../../../crypto/rand" } mc-transaction-core = { path = "../../../transaction/core" } mc-util-grpc = { path = "../../../util/grpc" } +mc-util-serial = { path = "../../../util/serial" } mc-util-uri = { path = "../../../util/uri" } # fog @@ -25,10 +30,13 @@ mc-fog-types = { path = "../../types" } mc-fog-uri = { path = "../../uri" } # third-party +aes-gcm = "0.10.1" displaydoc = { version = "0.2", default-features = false } grpcio = "0.12.1" +futures = "0.3" protobuf = "2.27.1" retry = "2.0" +sha2 = { version = "0.10", default-features = false } [dev-dependencies] mc-common = { path = "../../../common", features = ["loggers"] } diff --git a/fog/ledger/connection/src/lib.rs b/fog/ledger/connection/src/lib.rs index 0212496c1f..538c3fc7a4 100644 --- a/fog/ledger/connection/src/lib.rs +++ b/fog/ledger/connection/src/lib.rs @@ -18,3 +18,6 @@ pub use merkle_proof::{FogMerkleProofGrpcClient, OutputError, OutputResultExtens mod untrusted; pub use untrusted::FogUntrustedLedgerGrpcClient; + +mod router_client; +pub use router_client::LedgerGrpcClient; diff --git a/fog/ledger/connection/src/router_client.rs b/fog/ledger/connection/src/router_client.rs new file mode 100644 index 0000000000..38270e9968 --- /dev/null +++ b/fog/ledger/connection/src/router_client.rs @@ -0,0 +1,259 @@ +// Copyright (c) 2023 The MobileCoin Foundation + +use aes_gcm::Aes256Gcm; +use futures::{executor::block_on, SinkExt, TryStreamExt}; +use grpcio::{ChannelBuilder, ClientDuplexReceiver, ClientDuplexSender, Environment}; +use mc_attest_ake::{ + AuthResponseInput, ClientInitiate, Error as AttestAkeError, Ready, Start, Transition, +}; +use mc_attest_core::VerificationReport; +use mc_attest_verifier::Verifier; +use mc_common::logger::{log, o, Logger}; +use mc_crypto_keys::X25519; +use mc_crypto_noise::CipherError; +use mc_crypto_rand::McRng; +use mc_fog_api::{ + attest::{AuthMessage, Message}, + ledger::{LedgerRequest, LedgerResponse}, + ledger_grpc::LedgerApiClient, +}; +use mc_fog_types::ledger::{CheckKeyImagesRequest, CheckKeyImagesResponse, KeyImageQuery}; +use mc_fog_uri::FogLedgerUri; +use mc_transaction_core::ring_signature::KeyImage; +use mc_util_grpc::ConnectionUriGrpcioChannel; +use mc_util_serial::DecodeError; +use mc_util_uri::{ConnectionUri, UriConversionError}; +use sha2::Sha512; +use std::sync::Arc; + +/// A high-level object mediating requests to the fog ledger router service +pub struct LedgerGrpcClient { + /// A logger object + logger: Logger, + + /// The URI of the router to communicate with + uri: FogLedgerUri, + + /// An object which can verify a fog node's provided IAS report + verifier: Verifier, + + /// The AKE state machine object, if one is available. + attest_cipher: Option>, + + /// Sends requests to the fog ledger router + request_sender: ClientDuplexSender, + + /// Receives responses from the fog ledger router + response_receiver: ClientDuplexReceiver, + + /// Low-lever ledger API client + _client: LedgerApiClient, +} + +impl LedgerGrpcClient { + /// Creates a new fog ledger router grpc client and opens a streaming + /// connection to the fog ledger router service. + /// + /// Arguments: + /// * uri: The Uri to connect to + /// * verifier: The attestation verifier + /// * env: A grpc environment (thread pool) to use for this connection + /// * logger: For logging + pub fn new( + uri: FogLedgerUri, + verifier: Verifier, + env: Arc, + logger: Logger, + ) -> Self { + let logger = logger.new(o!("mc.fog.ledger.router.uri" => uri.to_string())); + + let ch = ChannelBuilder::default_channel_builder(env).connect_to_uri(&uri, &logger); + let client = LedgerApiClient::new(ch); + let (request_sender, response_receiver) = client + .request() + .expect("Could not retrieve grpc sender and receiver."); + + Self { + logger, + attest_cipher: None, + _client: client, + request_sender, + response_receiver, + uri, + verifier, + } + } + + fn is_attested(&self) -> bool { + self.attest_cipher.is_some() + } + + async fn attest(&mut self) -> Result { + // If we have an existing attestation, nuke it. + self.deattest(); + + let mut csprng = McRng::default(); + + let initiator = Start::new(self.uri.responder_id()?.to_string()); + + let init_input = ClientInitiate::::default(); + let (initiator, auth_request_output) = initiator.try_next(&mut csprng, init_input)?; + + let attested_message: AuthMessage = auth_request_output.into(); + let mut request = LedgerRequest::new(); + request.set_auth(attested_message); + self.request_sender + .send((request.clone(), grpcio::WriteFlags::default())) + .await?; + + let mut response = self + .response_receiver + .try_next() + .await? + .ok_or(Error::ResponseNotReceived)?; + let auth_response_msg = response.take_auth(); + + // Process server response, check if key exchange is successful + let auth_response_event = + AuthResponseInput::new(auth_response_msg.into(), self.verifier.clone()); + let (initiator, verification_report) = + initiator.try_next(&mut csprng, auth_response_event)?; + + self.attest_cipher = Some(initiator); + + Ok(verification_report) + } + + fn deattest(&mut self) { + if self.is_attested() { + log::trace!(self.logger, "Tearing down existing attested connection."); + self.attest_cipher = None; + } + } + + /// Check one or more key images against the ledger router service + pub async fn check_key_images( + &mut self, + key_images: &[KeyImage], + ) -> Result { + log::trace!(self.logger, "Check key images was called"); + if !self.is_attested() { + let verification_report = self.attest().await; + verification_report?; + } + + let key_images_queries = key_images + .iter() + .map(|&key_image| KeyImageQuery { + key_image, + start_block: 0, + }) + .collect(); + let key_images_request = CheckKeyImagesRequest { + queries: key_images_queries, + }; + + // No authenticated data associated with ledger query + let aad = vec![]; + + let msg = { + let attest_cipher = self + .attest_cipher + .as_mut() + .expect("no enclave_connection even though attest succeeded"); + + let mut msg = Message::new(); + msg.set_channel_id(Vec::from(attest_cipher.binding())); + msg.set_aad(aad.clone()); + + let plaintext_bytes = mc_util_serial::encode(&key_images_request); + + let request_ciphertext = attest_cipher.encrypt(&aad, &plaintext_bytes)?; + msg.set_data(request_ciphertext); + msg + }; + let mut request = LedgerRequest::new(); + request.set_check_key_images(msg); + + self.request_sender + .send((request.clone(), grpcio::WriteFlags::default())) + .await?; + + let message = self + .response_receiver + .try_next() + .await? + .ok_or(Error::ResponseNotReceived)? + .take_check_key_image_response(); + + { + let attest_cipher = self + .attest_cipher + .as_mut() + .expect("no enclave_connection even though attest succeeded"); + + let plaintext_bytes = attest_cipher.decrypt(message.get_aad(), message.get_data())?; + let plaintext_response: CheckKeyImagesResponse = + mc_util_serial::decode(&plaintext_bytes)?; + Ok(plaintext_response) + } + } +} + +impl Drop for LedgerGrpcClient { + fn drop(&mut self) { + block_on(self.request_sender.close()).expect("Couldn't close the router request sender"); + } +} + +/// Errors related to the Fog View Router Client. +#[derive(Debug)] +pub enum Error { + /// Decode errors. + Decode(DecodeError), + + /// Uri conversion errors. + UriConversion(UriConversionError), + + /// Cipher errors. + Cipher(CipherError), + + /// Attestation errors. + Attestation(AttestAkeError), + + /// Grpc errors. + Grpc(grpcio::Error), + + /// Response not received + ResponseNotReceived, +} + +impl From for Error { + fn from(err: DecodeError) -> Self { + Self::Decode(err) + } +} + +impl From for Error { + fn from(err: CipherError) -> Self { + Self::Cipher(err) + } +} + +impl From for Error { + fn from(err: grpcio::Error) -> Self { + Self::Grpc(err) + } +} + +impl From for Error { + fn from(err: UriConversionError) -> Self { + Self::UriConversion(err) + } +} + +impl From for Error { + fn from(err: AttestAkeError) -> Self { + Self::Attestation(err) + } +} diff --git a/fog/ledger/enclave/impl/src/lib.rs b/fog/ledger/enclave/impl/src/lib.rs index 7955475ea9..8bd870d0a1 100644 --- a/fog/ledger/enclave/impl/src/lib.rs +++ b/fog/ledger/enclave/impl/src/lib.rs @@ -354,7 +354,7 @@ where &self, auth_request: NonceAuthRequest, ) -> Result<(NonceAuthResponse, NonceSession)> { - self.ake.frontend_accept(auth_request).map_err(|e| e.into()) + Ok(self.ake.frontend_accept(auth_request)?) } } diff --git a/fog/ledger/server/src/bin/key_image_store.rs b/fog/ledger/server/src/bin/key_image_store.rs index e55b7ff7ef..c893ab64a0 100644 --- a/fog/ledger/server/src/bin/key_image_store.rs +++ b/fog/ledger/server/src/bin/key_image_store.rs @@ -2,6 +2,7 @@ use clap::Parser; use grpcio::{RpcStatus, RpcStatusCode}; +use mc_attest_net::{Client, RaClient}; use mc_common::{logger::log, time::SystemTimeProvider}; use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; use mc_fog_ledger_server::{KeyImageStoreServer, LedgerStoreConfig, ShardingStrategy}; @@ -40,10 +41,13 @@ fn main() { let watcher = WatcherDB::open_ro(&config.watcher_db, logger.clone()).expect("Could not open watcher DB"); + let ias_client = Client::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut store_server = match config.sharding_strategy.clone() { ShardingStrategy::Epoch(sharding_strategy) => KeyImageStoreServer::new_from_config( config.clone(), enclave, + ias_client, db, watcher, sharding_strategy, diff --git a/fog/ledger/server/src/bin/router.rs b/fog/ledger/server/src/bin/router.rs index 3daa1d9e7b..2209d3e7c0 100644 --- a/fog/ledger/server/src/bin/router.rs +++ b/fog/ledger/server/src/bin/router.rs @@ -9,6 +9,7 @@ use std::{ use clap::Parser; use grpcio::ChannelBuilder; +use mc_attest_net::{Client, RaClient}; use mc_common::logger::log; use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; @@ -83,9 +84,11 @@ fn main() { let watcher_db = WatcherDB::open_ro(&config.watcher_db, logger.clone()).expect("Could not open watcher DB"); + let ias_client = Client::new(&config.ias_api_key).expect("Could not create IAS client"); let mut router_server = LedgerRouterServer::new( config, enclave, + ias_client, ledger_store_grpc_clients, ledger_db, watcher_db, diff --git a/fog/ledger/server/src/config.rs b/fog/ledger/server/src/config.rs index 2dc6ce98de..fb2ea226c3 100644 --- a/fog/ledger/server/src/config.rs +++ b/fog/ledger/server/src/config.rs @@ -93,6 +93,14 @@ pub struct LedgerRouterConfig { #[clap(long, env = "MC_CLIENT_RESPONDER_ID")] pub client_responder_id: ResponderId, + /// PEM-formatted keypair to send with an Attestation Request. + #[clap(long, env = "MC_IAS_API_KEY")] + pub ias_api_key: String, + + /// The IAS SPID to use when getting a quote + #[clap(long, env = "MC_IAS_SPID")] + pub ias_spid: ProviderId, + /// gRPC listening URI for client requests. #[clap(long, env = "MC_CLIENT_LISTEN_URI")] pub client_listen_uri: FogLedgerUri, diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs deleted file mode 100644 index bce08d6dd8..0000000000 --- a/fog/ledger/server/src/key_image_router_server.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use futures::executor::block_on; -use mc_common::logger::{log, Logger}; -use mc_fog_api::ledger_grpc; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; -use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; - -use crate::{ - config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService, - router_admin_service::LedgerRouterAdminService, -}; - -pub struct KeyImageRouterServer { - router_server: grpcio::Server, - admin_server: grpcio::Server, - logger: Logger, -} - -impl KeyImageRouterServer { - pub fn new( - config: LedgerRouterConfig, - enclave: E, - shards: Arc>>>, - logger: Logger, - ) -> KeyImageRouterServer - where - E: LedgerEnclaveProxy, - { - let readiness_indicator = ReadinessIndicator::default(); - - let env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("key-image-router-server".to_string()) - .build(), - ); - - // Health check service - will be used in both cases - let health_service = - mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) - .into_service(); - - // Build our router server. - // Init ledger router service. - let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( - enclave, - shards.clone(), - logger.clone(), - )); - log::debug!(logger, "Constructed Key Image Router GRPC Service"); - - // Init ledger router admin service. - let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( - LedgerRouterAdminService::new(shards, logger.clone()), - ); - log::debug!(logger, "Constructed Key Image Router Admin GRPC Service"); - - // Package service into grpc server - log::info!( - logger, - "Starting Key Image Router server on {}", - config.client_listen_uri.addr(), - ); - - let router_server_builder = grpcio::ServerBuilder::new(env.clone()) - .register_service(ledger_router_service) - .register_service(health_service) - .bind_using_uri(&config.client_listen_uri, logger.clone()); - let admin_server_builder = grpcio::ServerBuilder::new(env) - .register_service(ledger_router_admin_service) - .bind_using_uri(&config.admin_listen_uri, logger.clone()); - - let router_server = router_server_builder.build().unwrap(); - let admin_server = admin_server_builder.build().unwrap(); - - Self { - router_server, - admin_server, - logger, - } - } - - /// Starts the server - pub fn start(&mut self) { - self.router_server.start(); - for (host, port) in self.router_server.bind_addrs() { - log::info!(self.logger, "Router API listening on {}:{}", host, port); - } - self.admin_server.start(); - for (host, port) in self.admin_server.bind_addrs() { - log::info!( - self.logger, - "Router Admin API listening on {}:{}", - host, - port - ); - } - } - - /// Stops the server - pub fn stop(&mut self) { - block_on(self.router_server.shutdown()).expect("Could not stop router grpc server"); - block_on(self.admin_server.shutdown()).expect("Could not stop router admin grpc server"); - } -} - -impl Drop for KeyImageRouterServer { - fn drop(&mut self) { - self.stop(); - } -} diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs deleted file mode 100644 index 707674547a..0000000000 --- a/fog/ledger/server/src/key_image_router_service.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use futures::{FutureExt, TryFutureExt}; -use grpcio::{DuplexSink, RequestStream, RpcContext}; -use mc_common::logger::{log, Logger}; -use mc_fog_api::{ - ledger::{LedgerRequest, LedgerResponse}, - ledger_grpc::{self, LedgerApi}, -}; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::KeyImageStoreUri; -use mc_util_grpc::rpc_logger; -use mc_util_metrics::SVC_COUNTERS; - -use crate::router_handlers; - -#[derive(Clone)] -pub struct KeyImageRouterService -where - E: LedgerEnclaveProxy, -{ - enclave: E, - shards: Arc>>>, - query_retries: usize, - logger: Logger, -} - -impl KeyImageRouterService { - /// Creates a new LedgerRouterService that can be used by a gRPC server to - /// fulfill gRPC requests. - #[allow(dead_code)] // FIXME - pub fn new( - enclave: E, - shards: Arc>>>, - query_retries: usize, - logger: Logger, - ) -> Self { - Self { - enclave, - shards, - query_retries, - logger, - } - } -} - -impl LedgerApi for KeyImageRouterService -where - E: LedgerEnclaveProxy, -{ - fn request( - &mut self, - ctx: RpcContext, - requests: RequestStream, - responses: DuplexSink, - ) { - let _timer = SVC_COUNTERS.req(&ctx); - mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { - log::warn!( - self.logger, - "Streaming GRPC Ledger API only partially implemented." - ); - let logger = logger.clone(); - - let shards = self.shards.read().expect("RwLock poisoned"); - let future = router_handlers::handle_requests( - shards.values().cloned().collect(), - self.enclave.clone(), - requests, - responses, - self.query_retries, - logger.clone(), - ) - .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) - // TODO: Do more with the error than just push it to the log. - .map(|_| ()); - - ctx.spawn(future) - }); - } -} diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index a23deaa957..ed9f39e1ea 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -73,7 +73,36 @@ impl KeyImageService { self.db_poll_shared_state.clone() } - pub fn auth_impl( + pub fn auth_store( + &mut self, + mut req: AuthMessage, + logger: &Logger, + ) -> Result { + // TODO: Use the prost message directly, once available + match self.enclave.frontend_accept(req.take_data().into()) { + Ok((response, _)) => { + let mut result = attest::AuthMessage::new(); + result.set_data(response.into()); + Ok(result) + } + Err(client_error) => { + // There's no requirement on the remote party to trigger this, so it's debug. + log::debug!( + logger, + "KeyImageStoreApi::frontend_accept failed: {}", + client_error + ); + let rpc_permissions_error = rpc_permissions_error( + "auth_store", + format!("Permission denied: {}", client_error), + logger, + ); + Err(rpc_permissions_error) + } + } + } + + pub fn auth_service( &mut self, mut req: AuthMessage, logger: &Logger, @@ -247,7 +276,7 @@ impl FogKeyImageApi for KeyImageServic return send_result(ctx, sink, err.into(), logger); } - match self.auth_impl(request, logger) { + match self.auth_service(request, logger) { Ok(response) => { send_result(ctx, sink, Ok(response), logger); } @@ -280,7 +309,7 @@ impl KeyImageStoreApi for KeyImageServ return send_result(ctx, sink, err.into(), logger); } - match self.auth_impl(req, logger) { + match self.auth_store(req, logger) { Ok(response) => { send_result(ctx, sink, Ok(response), logger); } @@ -289,7 +318,7 @@ impl KeyImageStoreApi for KeyImageServ // it. log::info!( logger, - "LedgerEnclave::client_accept failed: {}", + "LedgerEnclave::frontend_accept failed: {}", client_error ); // TODO: increment failed inbound peering counter. diff --git a/fog/ledger/server/src/key_image_store_server.rs b/fog/ledger/server/src/key_image_store_server.rs index d07090c139..552e251773 100644 --- a/fog/ledger/server/src/key_image_store_server.rs +++ b/fog/ledger/server/src/key_image_store_server.rs @@ -3,6 +3,8 @@ use std::sync::{Arc, Mutex}; use futures::executor::block_on; +use mc_attest_core::ProviderId; +use mc_attest_net::RaClient; use mc_common::{ logger::{log, Logger}, time::TimeProvider, @@ -11,6 +13,7 @@ use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; use mc_ledger_db::LedgerDB; +use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::{ AnonymousAuthenticator, Authenticator, ConnectionUriGrpcioServer, ReadinessIndicator, TokenAuthenticator, @@ -18,36 +21,43 @@ use mc_util_grpc::{ use mc_watcher::watcher_db::WatcherDB; use crate::{ - config::LedgerStoreConfig, db_fetcher::DbFetcher, server::DbPollSharedState, + config::LedgerStoreConfig, counters, db_fetcher::DbFetcher, server::DbPollSharedState, sharding_strategy::ShardingStrategy, KeyImageClientListenUri, KeyImageService, }; -pub struct KeyImageStoreServer +pub struct KeyImageStoreServer where E: LedgerEnclaveProxy, SS: ShardingStrategy + Send + Sync + 'static, + RC: RaClient + Send + Sync + 'static, { server: grpcio::Server, client_listen_uri: KeyImageStoreUri, db_fetcher: DbFetcher, + enclave: E, + ra_client: RC, + report_cache_thread: Option, + ias_spid: ProviderId, logger: Logger, } -impl KeyImageStoreServer +impl KeyImageStoreServer where E: LedgerEnclaveProxy, SS: ShardingStrategy + Send + Sync + 'static, + RC: RaClient + Send + Sync + 'static, { /// Creates a new key image store server instance pub fn new_from_config( config: LedgerStoreConfig, enclave: E, + ra_client: RC, ledger: LedgerDB, watcher: WatcherDB, sharding_strategy: SS, time_provider: impl TimeProvider + 'static, logger: Logger, - ) -> KeyImageStoreServer { + ) -> KeyImageStoreServer { let client_authenticator: Arc = if let Some(shared_secret) = config.client_auth_token_secret.as_ref() { Arc::new(TokenAuthenticator::new( @@ -64,6 +74,8 @@ where client_authenticator, config.client_listen_uri, enclave, + ra_client, + config.ias_spid, ledger, watcher, sharding_strategy, @@ -76,11 +88,13 @@ where client_authenticator: Arc, client_listen_uri: KeyImageStoreUri, enclave: E, + ra_client: RC, + ias_spid: ProviderId, ledger: LedgerDB, watcher: WatcherDB, sharding_strategy: SS, logger: Logger, - ) -> KeyImageStoreServer { + ) -> KeyImageStoreServer { let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); let key_image_service = KeyImageService::new( @@ -97,6 +111,8 @@ where key_image_service, client_listen_uri, enclave, + ra_client, + ias_spid, sharding_strategy, logger, ) @@ -106,9 +122,11 @@ where mut key_image_service: KeyImageService, client_listen_uri: KeyImageStoreUri, enclave: E, + ra_client: RC, + ias_spid: ProviderId, sharding_strategy: SS, logger: Logger, - ) -> KeyImageStoreServer { + ) -> KeyImageStoreServer { let readiness_indicator = ReadinessIndicator::default(); let env = Arc::new( @@ -145,7 +163,7 @@ where let db_fetcher = DbFetcher::new( key_image_service.get_ledger(), - enclave, + enclave.clone(), sharding_strategy, key_image_service.get_watcher(), key_image_service.get_db_poll_shared_state(), @@ -157,12 +175,27 @@ where server, client_listen_uri, db_fetcher, + enclave, + ra_client, + ias_spid, + report_cache_thread: None, logger, } } /// Starts the server pub fn start(&mut self) { + self.report_cache_thread = Some( + ReportCacheThread::start( + self.enclave.clone(), + self.ra_client.clone(), + self.ias_spid, + &counters::ENCLAVE_REPORT_TIMESTAMP, + self.logger.clone(), + ) + .expect("failed starting report cache thread"), + ); + self.server.start(); log::info!( self.logger, @@ -179,10 +212,11 @@ where } } -impl Drop for KeyImageStoreServer +impl Drop for KeyImageStoreServer where E: LedgerEnclaveProxy, SS: ShardingStrategy + Send + Sync + 'static, + RC: RaClient + Send + Sync + 'static, { fn drop(&mut self) { self.stop(); diff --git a/fog/ledger/server/src/router_handlers.rs b/fog/ledger/server/src/router_handlers.rs index c71df6064c..11adc457aa 100644 --- a/fog/ledger/server/src/router_handlers.rs +++ b/fog/ledger/server/src/router_handlers.rs @@ -156,7 +156,7 @@ pub fn process_shard_responses( } /// Handles a client's authentication request. -fn handle_auth_request( +pub(crate) fn handle_auth_request( enclave: E, auth_message: attest::AuthMessage, logger: Logger, @@ -174,7 +174,7 @@ where } /// Handles a client's query request. -async fn handle_query_request( +pub(crate) async fn handle_query_request( query: attest::Message, enclave: E, shard_clients: Vec>, @@ -332,7 +332,7 @@ async fn authenticate_ledger_store( ledger_store_url: KeyImageStoreUri, logger: Logger, ) -> Result<(), RouterServerError> { - let ledger_store_id = ResponderId::from_str(&ledger_store_url.to_string())?; + let ledger_store_id = ledger_store_url.responder_id()?; let client_auth_request = enclave.ledger_store_init(ledger_store_id.clone())?; let grpc_env = Arc::new( grpcio::EnvBuilder::new() diff --git a/fog/ledger/server/src/router_server.rs b/fog/ledger/server/src/router_server.rs index 98af9eb8f0..03b01d32d5 100644 --- a/fog/ledger/server/src/router_server.rs +++ b/fog/ledger/server/src/router_server.rs @@ -6,6 +6,7 @@ use std::{ }; use futures::executor::block_on; +use mc_attest_net::RaClient; use mc_common::{ logger::{log, Logger}, time::SystemTimeProvider, @@ -14,6 +15,7 @@ use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::{ConnectionUri, FogLedgerUri, KeyImageStoreUri}; use mc_ledger_db::LedgerDB; +use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::{ AnonymousAuthenticator, Authenticator, ConnectionUriGrpcioServer, ReadinessIndicator, TokenAuthenticator, @@ -22,27 +24,40 @@ use mc_util_uri::AdminUri; use mc_watcher::watcher_db::WatcherDB; use crate::{ - config::LedgerRouterConfig, router_admin_service::LedgerRouterAdminService, + config::LedgerRouterConfig, counters, router_admin_service::LedgerRouterAdminService, router_service::LedgerRouterService, BlockService, MerkleProofService, UntrustedTxOutService, }; -pub struct LedgerRouterServer { +pub struct LedgerRouterServer +where + E: LedgerEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ router_server: grpcio::Server, admin_server: grpcio::Server, client_listen_uri: FogLedgerUri, admin_listen_uri: AdminUri, + config: LedgerRouterConfig, + enclave: E, + ra_client: RC, + report_cache_thread: Option, logger: Logger, } -impl LedgerRouterServer { - pub fn new( +impl LedgerRouterServer +where + E: LedgerEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ + pub fn new( config: LedgerRouterConfig, enclave: E, + ra_client: RC, shards: Arc>>>, ledger: LedgerDB, watcher: WatcherDB, logger: Logger, - ) -> LedgerRouterServer + ) -> LedgerRouterServer where E: LedgerEnclaveProxy, { @@ -72,14 +87,18 @@ impl LedgerRouterServer { // Build our router server. // Init ledger router service. - let ledger_router_service = ledger_grpc::create_ledger_api(LedgerRouterService::new( + let ledger_service = LedgerRouterService::new( enclave.clone(), shards.clone(), config.query_retries, logger.clone(), - )); + ); + + let ledger_router_service = ledger_grpc::create_ledger_api(ledger_service.clone()); log::debug!(logger, "Constructed Ledger Router GRPC Service"); + let unary_key_image_service = ledger_grpc::create_fog_key_image_api(ledger_service); + // Init ledger router admin service. let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( LedgerRouterAdminService::new(shards, logger.clone()), @@ -92,7 +111,7 @@ impl LedgerRouterServer { ledger_grpc::create_fog_merkle_proof_api(MerkleProofService::new( config.chain_id.clone(), ledger.clone(), - enclave, + enclave.clone(), client_authenticator.clone(), logger.clone(), )); @@ -123,6 +142,7 @@ impl LedgerRouterServer { let router_server = grpcio::ServerBuilder::new(env.clone()) .register_service(ledger_router_service) + .register_service(unary_key_image_service) .register_service(merkle_proof_service) .register_service(untrusted_tx_out_service) .register_service(block_service) @@ -137,14 +157,29 @@ impl LedgerRouterServer { Self { router_server, admin_server, - client_listen_uri: config.client_listen_uri, - admin_listen_uri: config.admin_listen_uri, + client_listen_uri: config.client_listen_uri.clone(), + admin_listen_uri: config.admin_listen_uri.clone(), + config, + enclave, + ra_client, + report_cache_thread: None, logger, } } /// Starts the server pub fn start(&mut self) { + self.report_cache_thread = Some( + ReportCacheThread::start( + self.enclave.clone(), + self.ra_client.clone(), + self.config.ias_spid, + &counters::ENCLAVE_REPORT_TIMESTAMP, + self.logger.clone(), + ) + .expect("failed starting report cache thread"), + ); + self.router_server.start(); log::info!( self.logger, @@ -167,7 +202,11 @@ impl LedgerRouterServer { } } -impl Drop for LedgerRouterServer { +impl Drop for LedgerRouterServer +where + E: LedgerEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ fn drop(&mut self) { self.stop(); } diff --git a/fog/ledger/server/src/router_service.rs b/fog/ledger/server/src/router_service.rs index de61d241ca..c3b5983c82 100644 --- a/fog/ledger/server/src/router_service.rs +++ b/fog/ledger/server/src/router_service.rs @@ -1,16 +1,20 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use crate::{router_handlers, SVC_COUNTERS}; +use crate::{ + router_handlers::{self, handle_auth_request, handle_query_request}, + SVC_COUNTERS, +}; use futures::{FutureExt, TryFutureExt}; -use grpcio::{DuplexSink, RequestStream, RpcContext}; +use grpcio::{DuplexSink, RequestStream, RpcContext, UnarySink}; +use mc_attest_api::attest::{AuthMessage, Message}; use mc_common::logger::{log, Logger}; use mc_fog_api::{ ledger::{LedgerRequest, LedgerResponse}, - ledger_grpc::{self, LedgerApi}, + ledger_grpc::{self, FogKeyImageApi, KeyImageStoreApiClient, LedgerApi}, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::KeyImageStoreUri; -use mc_util_grpc::rpc_logger; +use mc_util_grpc::{rpc_internal_error, rpc_logger}; use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -80,3 +84,95 @@ where }); } } + +/// Used for the implementation of FogKeyImageApi::check_key_images(), +/// the legacy unary key-image API, for LedgerRouterService. +async fn unary_check_key_image_impl( + request: Message, + query_retries: usize, + enclave: E, + sink: UnarySink, + shard_clients: Vec>, + scope_logger: Logger, +) -> Result<(), grpcio::Error> +where + E: LedgerEnclaveProxy, +{ + let result = handle_query_request( + request, + enclave, + shard_clients, + query_retries, + scope_logger.clone(), + ) + .await; + + match result { + Ok(mut response) => { + if response.has_check_key_image_response() { + sink.success(response.take_check_key_image_response()).await + } else { + let error = rpc_internal_error( + "Inavlid LedgerRequest response", + "Cannot provide a check key image response to the client's key image request." + .to_string(), + &scope_logger, + ); + sink.fail(error).await + } + } + Err(rpc_status) => sink.fail(rpc_status).await, + } +} + +// This API is the unary key-image-specific equivalent of LedgerApi. +impl FogKeyImageApi for LedgerRouterService { + fn check_key_images(&mut self, ctx: RpcContext, request: Message, sink: UnarySink) { + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + let logger = logger.clone(); + let shards = self.shards.read().expect("RwLock poisoned"); + + let future = unary_check_key_image_impl( + request, + self.query_retries, + self.enclave.clone(), + sink, + shards.values().cloned().collect(), + logger.clone(), + ) + .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) + // TODO: Do more with the error than just push it to the log. + .map(|_| ()); + + ctx.spawn(future); + }) + } + + fn auth(&mut self, ctx: RpcContext, request: AuthMessage, sink: UnarySink) { + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + let logger = logger.clone(); + let result = handle_auth_request(self.enclave.clone(), request, logger.clone()); + let future = match result { + Ok(mut response) => { + if response.has_auth() { + sink.success(response.take_auth()) + } else { + let error = rpc_internal_error( + "Inavlid LedgerRequest response", + "Response to client's auth request did not contain an auth response." + .to_string(), + &logger, + ); + sink.fail(error) + } + } + Err(rpc_status) => sink.fail(rpc_status), + } + .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) + .map(|_| ()); + ctx.spawn(future); + }); + } +} diff --git a/fog/ledger/server/tests/connection.rs b/fog/ledger/server/tests/connection.rs index b1066ec98c..0204e52d15 100644 --- a/fog/ledger/server/tests/connection.rs +++ b/fog/ledger/server/tests/connection.rs @@ -59,8 +59,6 @@ fn setup_watcher_db(logger: Logger) -> (WatcherDB, PathBuf) { // hitting a fog ledger server #[test_with_logger] fn fog_ledger_merkle_proofs_test(logger: Logger) { - let base_port = 3230; - let mut rng = RngType::from_seed([0u8; 32]); for block_version in BlockVersion::iterator() { @@ -111,7 +109,7 @@ fn fog_ledger_merkle_proofs_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + portpicker::pick_unused_port().expect("No free ports"), )) .unwrap(); let config = LedgerServerConfig { @@ -267,8 +265,6 @@ fn fog_ledger_merkle_proofs_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_key_images_test(logger: Logger) { - let base_port = 3240; - let mut rng = RngType::from_seed([0u8; 32]); for block_version in BlockVersion::iterator() { @@ -341,7 +337,7 @@ fn fog_ledger_key_images_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + portpicker::pick_unused_port().expect("No free ports") )) .unwrap(); let config = LedgerServerConfig { @@ -483,8 +479,6 @@ fn fog_ledger_key_images_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_blocks_api_test(logger: Logger) { - let base_port = 3250; - let mut rng = RngType::from_seed([0u8; 32]); let alice = AccountKey::random_with_fog(&mut rng); @@ -543,7 +537,7 @@ fn fog_ledger_blocks_api_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + portpicker::pick_unused_port().expect("No free ports") )) .unwrap(); let config = LedgerServerConfig { @@ -641,8 +635,6 @@ fn fog_ledger_blocks_api_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { - let base_port = 3260; - let mut rng = RngType::from_seed([0u8; 32]); let alice = AccountKey::random_with_fog(&mut rng); @@ -701,7 +693,7 @@ fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + portpicker::pick_unused_port().expect("No free ports") )) .unwrap(); let config = LedgerServerConfig { diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs new file mode 100644 index 0000000000..806302531e --- /dev/null +++ b/fog/ledger/server/tests/router_connection.rs @@ -0,0 +1,914 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +//! Integration tests at the level of the fog ledger connection / fog ledger +//! grpc API + +use futures::executor::block_on; +use grpcio::ChannelBuilder; +use mc_account_keys::{AccountKey, PublicAddress}; +use mc_api::watcher::TimestampResultCode; +use mc_attest_net::{Client as AttestClient, RaClient}; +use mc_attest_verifier::{MrSignerVerifier, Verifier, DEBUG_ENCLAVE}; +use mc_blockchain_types::{BlockSignature, BlockVersion}; +use mc_common::{ + logger::{test_with_logger, Logger}, + time::SystemTimeProvider, +}; +use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; +use mc_fog_api::{ledger::TxOutResultCode, ledger_grpc::KeyImageStoreApiClient}; +use mc_fog_ledger_connection::{ + Error, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, KeyImageResultExtension, + LedgerGrpcClient, OutputResultExtension, +}; +use mc_fog_ledger_enclave::LedgerSgxEnclave; +use mc_fog_ledger_server::{ + sharding_strategy::EpochShardingStrategy, KeyImageStoreServer, LedgerRouterConfig, + LedgerRouterServer, LedgerStoreConfig, ShardingStrategy, +}; +use mc_fog_test_infra::get_enclave_path; +use mc_fog_uri::{ConnectionUri, FogLedgerUri, KeyImageStoreUri}; +use mc_ledger_db::{test_utils::recreate_ledger_db, Ledger, LedgerDB}; +use mc_transaction_core::{ + membership_proofs::compute_implied_merkle_root, ring_signature::KeyImage, tokens::Mob, Amount, + Token, +}; +use mc_util_from_random::FromRandom; +use mc_util_grpc::{ConnectionUriGrpcioChannel, GrpcRetryConfig, CHAIN_ID_MISMATCH_ERR_MSG}; +use mc_util_test_helper::{CryptoRng, RngCore, RngType, SeedableRng}; +use mc_util_uri::AdminUri; +use mc_watcher::watcher_db::WatcherDB; +use std::{ + collections::HashMap, + path::PathBuf, + str::FromStr, + sync::{Arc, RwLock}, + thread::sleep, + time::Duration, +}; +use tempdir::TempDir; +use url::Url; + +const TEST_URL: &str = "http://www.my_url1.com"; + +const OMAP_CAPACITY: u64 = 128 * 128; + +const GRPC_RETRY_CONFIG: GrpcRetryConfig = GrpcRetryConfig { + grpc_retry_count: 3, + grpc_retry_millis: 20, +}; + +fn setup_watcher_db(logger: Logger) -> (WatcherDB, PathBuf) { + let url = Url::parse(TEST_URL).unwrap(); + + let db_tmp = TempDir::new("wallet_db").expect("Could not make tempdir for wallet db"); + WatcherDB::create(db_tmp.path()).unwrap(); + let watcher = WatcherDB::open_rw(db_tmp.path(), &[url], logger).unwrap(); + let watcher_dir = db_tmp.path().to_path_buf(); + (watcher, watcher_dir) +} + +// Test that a fog ledger connection is able to get valid merkle proofs by +// hitting a fog ledger server +#[test_with_logger] +fn fog_ledger_merkle_proofs_test(logger: Logger) { + let mut rng = RngType::from_seed([0u8; 32]); + + for block_version in BlockVersion::iterator() { + let alice = AccountKey::random_with_fog(&mut rng); + let bob = AccountKey::random_with_fog(&mut rng); + let charlie = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![ + alice.default_subaddress(), + bob.default_subaddress(), + charlie.default_subaddress(), + ]; + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[KeyImage::from(1)], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[KeyImage::from(2)], + &mut rng, + &mut watcher, + ); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports"), + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri: admin_listen_uri.clone(), + client_listen_uri: client_listen_uri.clone(), + client_responder_id: client_listen_uri + .responder_id() + .expect("Couldn't get responder ID for router"), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make ledger enclave client + let mut mr_signer_verifier = + MrSignerVerifier::from(mc_fog_ledger_enclave_measurement::sigstruct()); + mr_signer_verifier.allow_hardening_advisories( + mc_fog_ledger_enclave_measurement::HARDENING_ADVISORIES, + ); + + let mut verifier = Verifier::default(); + verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); + + let mut client = FogMerkleProofGrpcClient::new( + "local".to_string(), + client_listen_uri.clone(), + GRPC_RETRY_CONFIG, + verifier.clone(), + grpc_env.clone(), + logger.clone(), + ); + + // Get merkle root of num_blocks - 1 + let merkle_root = { + let temp = ledger.get_tx_out_proof_of_memberships(&[0u64]).unwrap(); + let merkle_proof = &temp[0]; + compute_implied_merkle_root(merkle_proof).unwrap() + }; + + // Get some tx outs and merkle proofs + let response = client + .get_outputs( + vec![0u64, 1u64, 2u64, 3u64, 4u64, 5u64, 6u64, 7u64, 8u64], + num_blocks - 1, + ) + .expect("get outputs failed"); + + // Test the basic fields + assert_eq!(response.num_blocks, num_blocks); + assert_eq!(response.global_txo_count, ledger.num_txos().unwrap()); + + // Validate merkle proofs + for res in response.results.iter() { + let (tx_out, proof) = res.status().unwrap().unwrap(); + let result = mc_transaction_core::membership_proofs::is_membership_proof_valid( + &tx_out, + &proof, + merkle_root.hash.as_ref(), + ) + .expect("membership proof structure failed!"); + assert!(result, "membership proof was invalid! idx = {}, output = {:?}, proof = {:?}, merkle_root = {:?}", res.index, tx_out, proof, merkle_root); + } + + // Make some queries that are out of bounds + let response = client + .get_outputs(vec![1u64, 6u64, 9u64, 14u64], num_blocks - 1) + .expect("get outputs failed"); + + // Test the basic fields + assert_eq!(response.num_blocks, num_blocks); + assert_eq!(response.global_txo_count, ledger.num_txos().unwrap()); + assert_eq!(response.results.len(), 4); + assert!(response.results[0].status().as_ref().unwrap().is_some()); + assert!(response.results[1].status().as_ref().unwrap().is_some()); + assert!(response.results[2].status().as_ref().unwrap().is_none()); + assert!(response.results[3].status().as_ref().unwrap().is_none()); + + // Check that wrong chain id results in an error + let mut client = FogMerkleProofGrpcClient::new( + "wrong".to_string(), + client_listen_uri, + GRPC_RETRY_CONFIG, + verifier, + grpc_env, + logger.clone(), + ); + + let result = client.get_outputs( + vec![0u64, 1u64, 2u64, 3u64, 4u64, 5u64, 6u64, 7u64, 8u64], + num_blocks - 1, + ); + + if let Err(err) = result { + match err { + Error::Connection( + _, + retry::Error { + error: + mc_fog_enclave_connection::Error::Rpc(grpcio::Error::RpcFailure(status)), + .. + }, + ) => { + let expected = format!("{} '{}'", CHAIN_ID_MISMATCH_ERR_MSG, "local"); + assert_eq!(status.message(), expected); + } + _ => { + panic!("unexpected grpcio error: {}", err); + } + } + } else { + panic!("Expected an error when chain-id is wrong"); + } + } + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); + } +} + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger server +#[test_with_logger] +fn fog_ledger_key_images_test(logger: Logger) { + let mut rng = RngType::from_seed([0u8; 32]); + + for block_version in BlockVersion::iterator() { + let alice = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + let keys: Vec = (0..20).map(|x| KeyImage::from(x as u64)).collect(); + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + // Make WatcherDB + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[0..2], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[3..6], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[6..9], + &mut rng, + &mut watcher, + ); + + // Populate watcher with Signature and Timestamp for block 1 + let url1 = Url::parse(TEST_URL).unwrap(); + let block1 = ledger.get_block(1).unwrap(); + let signing_key_a = Ed25519Pair::from_random(&mut rng); + let filename = String::from("00/00"); + let mut signed_block_a1 = + BlockSignature::from_block_and_keypair(&block1, &signing_key_a).unwrap(); + signed_block_a1.set_signed_at(1593798844); + watcher + .add_block_signature(&url1, 1, signed_block_a1, filename.clone()) + .unwrap(); + + // Update last synced to block 2, to indicate that this URL did not participate + // in consensus for block 2. + watcher.update_last_synced(&url1, 2).unwrap(); + + { + // Make Key Image Store + let store_uri = KeyImageStoreUri::from_str(&format!( + "insecure-key-image-store://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let store_admin_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let store_config = LedgerStoreConfig { + chain_id: "local".to_string(), + client_responder_id: store_uri + .responder_id() + .expect("Couldn't get responder ID for store"), + client_listen_uri: store_uri.clone(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir.clone(), + ias_api_key: Default::default(), + ias_spid: Default::default(), + admin_listen_uri: Some(store_admin_uri), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + omap_capacity: OMAP_CAPACITY, + sharding_strategy: ShardingStrategy::Epoch(EpochShardingStrategy::default()), + }; + let store_enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &store_config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + let ra_client = + AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); + let mut store_server = KeyImageStoreServer::new_from_config( + store_config, + store_enclave, + ra_client, + ledger.clone(), + watcher.clone(), + EpochShardingStrategy::default(), + SystemTimeProvider::default(), + logger.clone(), + ); + + // Make Key Image Store client + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let store_client = KeyImageStoreApiClient::new( + ChannelBuilder::default_channel_builder(grpc_env.clone()) + .connect_to_uri(&store_uri, &logger), + ); + let mut store_clients = HashMap::new(); + store_clients.insert(store_uri, Arc::new(store_client)); + let shards = Arc::new(RwLock::new(store_clients)); + + // Make Router Server + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports"), + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let router_config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri: admin_listen_uri.clone(), + client_listen_uri: client_listen_uri.clone(), + client_responder_id: client_listen_uri + .responder_id() + .expect("Couldn't get responder ID for router"), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &router_config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let ra_client = + AttestClient::new(&router_config.ias_api_key).expect("Could not create IAS client"); + let mut router_server = LedgerRouterServer::new( + router_config, + enclave, + ra_client, + shards, + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + store_server.start(); + router_server.start(); + + // Make ledger enclave client + let mut mr_signer_verifier = + MrSignerVerifier::from(mc_fog_ledger_enclave_measurement::sigstruct()); + mr_signer_verifier.allow_hardening_advisories( + mc_fog_ledger_enclave_measurement::HARDENING_ADVISORIES, + ); + + let mut verifier = Verifier::default(); + verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); + + let mut client = + LedgerGrpcClient::new(client_listen_uri, verifier, grpc_env, logger.clone()); + + // Check on key images + let mut response = + block_on(client.check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]])) + .expect("check_key_images failed"); + + let mut n = 1; + // adding a delay to give fog ledger time to fully initialize + while response.num_blocks != num_blocks { + response = block_on( + client.check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]), + ) + .expect("check_key_images failed"); + + sleep(Duration::from_secs(10)); + // panic on the 20th time + n += 1; // + if n > 20 { + panic!("Fog ledger not fully initialized"); + } + } + + // FIXME assert_eq!(response.num_txos, ...); + assert_eq!(response.results[0].key_image, keys[0]); + assert_eq!(response.results[0].status(), Ok(Some(1))); + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[0].timestamp, 1); + + assert_eq!(response.results[1].key_image, keys[1]); + assert_eq!(response.results[1].status(), Ok(Some(1))); + assert_eq!( + response.results[1].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[1].timestamp, 1); + + // Check a key_image for a block which will never have signatures & timestamps + assert_eq!(response.results[2].key_image, keys[3]); + assert_eq!(response.results[2].status(), Ok(Some(2))); // Spent in block 2 + assert_eq!( + response.results[2].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[2].timestamp, 2); + + // Watcher has only synced 1 block, so timestamp should be behind + assert_eq!(response.results[3].key_image, keys[7]); + assert_eq!(response.results[3].status(), Ok(Some(3))); // Spent in block 3 + assert_eq!( + response.results[3].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[3].timestamp, 3); + + // Check a key_image that has not been spent + assert_eq!(response.results[4].key_image, keys[19]); + assert_eq!(response.results[4].status(), Ok(None)); // Not spent + assert_eq!( + response.results[4].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[4].timestamp, u64::MAX); + } + + // FIXME: Check a key_image that generates a DatabaseError - tough to generate + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); + } +} + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger server +#[test_with_logger] +fn fog_ledger_blocks_api_test(logger: Logger) { + let mut rng = RngType::from_seed([0u8; 32]); + + let alice = AccountKey::random_with_fog(&mut rng); + let bob = AccountKey::random_with_fog(&mut rng); + let charlie = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress()], + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress(), bob.default_subaddress()], + &[KeyImage::from(1)], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[ + alice.default_subaddress(), + bob.default_subaddress(), + charlie.default_subaddress(), + ], + &[KeyImage::from(2)], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &recipients, + &[KeyImage::from(3)], + &mut rng, + &mut watcher, + ); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri, + client_listen_uri: client_listen_uri.clone(), + client_responder_id: client_listen_uri + .responder_id() + .expect("Couldn't get responder ID for router"), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make unattested ledger client + let client = FogUntrustedLedgerGrpcClient::new( + client_listen_uri, + GRPC_RETRY_CONFIG, + grpc_env, + logger.clone(), + ); + + // Try to get a block + let queries = [0..1]; + let result = client.get_blocks(&queries).unwrap(); + // Check that we got 1 block, as expected + assert_eq!(result.blocks.len(), 1); + assert_eq!(result.blocks[0].index, 0); + assert_eq!(result.blocks[0].outputs.len(), 1); + assert_eq!(result.blocks[0].global_txo_count, 1); + assert_eq!( + result.blocks[0].timestamp_result_code, + TimestampResultCode::BlockIndexOutOfBounds as u32 + ); + assert_eq!(result.num_blocks, num_blocks); + assert_eq!(result.global_txo_count, ledger.num_txos().unwrap()); + + // Try to get two blocks + let queries = [1..3]; + let result = client.get_blocks(&queries).unwrap(); + + // Check that we got 2 blocks, as expected + assert_eq!(result.blocks.len(), 2); + assert_eq!(result.blocks[0].index, 1); + assert_eq!(result.blocks[0].outputs.len(), 2); + assert_eq!(result.blocks[0].global_txo_count, 3); + assert_eq!( + result.blocks[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(result.blocks[1].index, 2); + assert_eq!(result.blocks[1].outputs.len(), 3); + assert_eq!(result.blocks[1].global_txo_count, 6); + assert_eq!( + result.blocks[1].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(result.num_blocks, num_blocks); + assert_eq!(result.global_txo_count, ledger.num_txos().unwrap()); + } + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); +} + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger server +#[test_with_logger] +fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { + let mut rng = RngType::from_seed([0u8; 32]); + + let alice = AccountKey::random_with_fog(&mut rng); + let bob = AccountKey::random_with_fog(&mut rng); + let charlie = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress()], + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress(), bob.default_subaddress()], + &[KeyImage::from(1)], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[ + alice.default_subaddress(), + bob.default_subaddress(), + charlie.default_subaddress(), + ], + &[KeyImage::from(2)], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &recipients, + &[KeyImage::from(3)], + &mut rng, + &mut watcher, + ); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + portpicker::pick_unused_port().expect("No free ports") + )) + .unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri, + client_listen_uri: client_listen_uri.clone(), + client_responder_id: client_listen_uri + .responder_id() + .expect("Couldn't get responder ID for router"), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make unattested ledger client + let client = FogUntrustedLedgerGrpcClient::new( + client_listen_uri, + GRPC_RETRY_CONFIG, + grpc_env, + logger.clone(), + ); + + // Get a tx_out that is actually in the ledger + let real_tx_out0 = { ledger.get_tx_out_by_index(0).unwrap() }; + + // Try to get tx out records + let queries: Vec = + vec![(&[0u8; 32]).into(), real_tx_out0.public_key]; + let result = client.get_tx_outs(queries).unwrap(); + // Check that we got expected num_blocks value + assert_eq!(result.num_blocks, 4); + // Check that we got 2 results, as expected + assert_eq!(result.results.len(), 2); + assert_eq!( + &result.results[0].tx_out_pubkey.clone().unwrap().data[..], + &[0u8; 32] + ); + assert_eq!(result.results[0].result_code, TxOutResultCode::NotFound); + assert_eq!( + &result.results[1].tx_out_pubkey.clone().unwrap().data[..], + &real_tx_out0.public_key.as_bytes()[..] + ); + assert_eq!(result.results[1].result_code, TxOutResultCode::Found); + assert_eq!(result.results[1].tx_out_global_index, 0); + assert_eq!(result.results[1].block_index, 0); + assert_eq!( + result.results[1].timestamp_result_code, + TimestampResultCode::BlockIndexOutOfBounds as u32 + ); + } + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); +} + +// Infra + +/// Adds a block containing one txo for each provided recipient and returns new +/// block height. +/// +/// # Arguments +/// * `ledger_db` +/// * `recipients` - Recipients of outputs. +/// * `rng` +fn add_block_to_ledger( + block_version: BlockVersion, + ledger_db: &mut LedgerDB, + recipients: &[PublicAddress], + key_images: &[KeyImage], + rng: &mut (impl CryptoRng + RngCore), + watcher: &mut WatcherDB, +) -> u64 { + let amount = Amount::new(10, Mob::ID); + let block_data = mc_ledger_db::test_utils::add_block_to_ledger( + ledger_db, + block_version, + recipients, + amount, + key_images, + rng, + ) + .expect("failed to add block"); + let block_index = block_data.block().index; + + let signature = block_data.signature().expect("missing signature"); + for src_url in watcher.get_config_urls().unwrap().iter() { + watcher + .add_block_signature( + src_url, + block_index, + signature.clone(), + format!("00/{}", block_index), + ) + .expect("Could not add block signature"); + } + + block_index + 1 +} diff --git a/fog/ledger/server/tests/store.rs b/fog/ledger/server/tests/store.rs index 9cc30e3027..aa1d4f1cb8 100644 --- a/fog/ledger/server/tests/store.rs +++ b/fog/ledger/server/tests/store.rs @@ -28,7 +28,6 @@ use mc_fog_ledger_server::{ use mc_fog_types::ledger::{CheckKeyImagesRequest, KeyImageQuery}; use mc_fog_uri::{ConnectionUri, KeyImageStoreScheme, KeyImageStoreUri}; use mc_ledger_db::{test_utils::recreate_ledger_db, LedgerDB}; -use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::AnonymousAuthenticator; use mc_util_metrics::{IntGauge, OpMetrics}; use mc_util_test_helper::{Rng, RngType, SeedableRng}; @@ -36,6 +35,7 @@ use mc_util_uri::UriScheme; use mc_watcher::watcher_db::WatcherDB; use aes_gcm::Aes256Gcm; +use portpicker::pick_unused_port; use sha2::Sha512; use tempdir::TempDir; use url::Url; @@ -151,9 +151,10 @@ lazy_static::lazy_static! { #[test_with_logger] pub fn direct_key_image_store_check(logger: Logger) { const TEST_NAME: &str = "direct_key_image_store_check"; - const PORT_START: u16 = 3223; const OMAP_CAPACITY: u64 = 768; + let port = pick_unused_port().expect("No free ports"); + let rng = RngType::from_entropy(); let TestingContext { enclave, @@ -165,7 +166,7 @@ pub fn direct_key_image_store_check(logger: Logger) { watcher, store_config, watcher_path: _watcher_path, - } = TestingContext::new(TEST_NAME, logger.clone(), PORT_START, OMAP_CAPACITY, rng); + } = TestingContext::new(TEST_NAME, logger.clone(), port, OMAP_CAPACITY, rng); let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); @@ -181,27 +182,20 @@ pub fn direct_key_image_store_check(logger: Logger) { logger.clone(), ); - let mut store_server = KeyImageStoreServer::new_from_service( - store_service, - client_listen_uri, - enclave.clone(), - EpochShardingStrategy::default(), - logger.clone(), - ); - store_server.start(); - // Set up IAS verficiation // This will be a SimClient in testing contexts. let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); - let _report_cache_thread = ReportCacheThread::start( + let mut store_server = KeyImageStoreServer::new_from_service( + store_service, + client_listen_uri, enclave.clone(), ias_client, store_config.ias_spid, - &TEST_ENCLAVE_REPORT_TIMESTAMP, + EpochShardingStrategy::default(), logger.clone(), - ) - .expect("Failed to start IAS client."); + ); + store_server.start(); // Make GRPC client for sending requests. diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index 73bf6a42df..92f0b93965 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -37,6 +37,7 @@ use mc_util_uri::UriScheme; use mc_watcher::watcher_db::WatcherDB; use aes_gcm::Aes256Gcm; +use portpicker::pick_unused_port; use sha2::Sha512; use tempdir::TempDir; use url::Url; @@ -152,10 +153,9 @@ lazy_static::lazy_static! { #[test_with_logger] pub fn direct_key_image_store_check(logger: Logger) { const TEST_NAME: &str = "direct_key_image_store_check"; - const PORT_START: u16 = 3223; const OMAP_CAPACITY: u64 = 768; - let port = PORT_START; + let port = pick_unused_port().expect("No free ports"); let rng = RngType::from_entropy(); let TestingContext { @@ -183,11 +183,15 @@ pub fn direct_key_image_store_check(logger: Logger) { Arc::new(AnonymousAuthenticator::default()), logger.clone(), ); + let ra_client = + AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); let mut store_server = KeyImageStoreServer::new_from_service( store_service, client_listen_uri, enclave.clone(), + ra_client, + store_config.ias_spid, EpochShardingStrategy::default(), logger.clone(), );