From 1765353d16cb68e5ab0e03d284305900bc5a23e9 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Thu, 9 Feb 2023 11:01:46 -0800 Subject: [PATCH 01/11] Port existing tests to router server binary, except key image --- fog/ledger/server/src/bin/router.rs | 3 + fog/ledger/server/src/config.rs | 8 + fog/ledger/server/src/router_server.rs | 52 +- fog/ledger/server/tests/router_connection.rs | 846 +++++++++++++++++++ 4 files changed, 900 insertions(+), 9 deletions(-) create mode 100644 fog/ledger/server/tests/router_connection.rs diff --git a/fog/ledger/server/src/bin/router.rs b/fog/ledger/server/src/bin/router.rs index 3daa1d9e7b..2209d3e7c0 100644 --- a/fog/ledger/server/src/bin/router.rs +++ b/fog/ledger/server/src/bin/router.rs @@ -9,6 +9,7 @@ use std::{ use clap::Parser; use grpcio::ChannelBuilder; +use mc_attest_net::{Client, RaClient}; use mc_common::logger::log; use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; @@ -83,9 +84,11 @@ fn main() { let watcher_db = WatcherDB::open_ro(&config.watcher_db, logger.clone()).expect("Could not open watcher DB"); + let ias_client = Client::new(&config.ias_api_key).expect("Could not create IAS client"); let mut router_server = LedgerRouterServer::new( config, enclave, + ias_client, ledger_store_grpc_clients, ledger_db, watcher_db, diff --git a/fog/ledger/server/src/config.rs b/fog/ledger/server/src/config.rs index 2dc6ce98de..fb2ea226c3 100644 --- a/fog/ledger/server/src/config.rs +++ b/fog/ledger/server/src/config.rs @@ -93,6 +93,14 @@ pub struct LedgerRouterConfig { #[clap(long, env = "MC_CLIENT_RESPONDER_ID")] pub client_responder_id: ResponderId, + /// PEM-formatted keypair to send with an Attestation Request. + #[clap(long, env = "MC_IAS_API_KEY")] + pub ias_api_key: String, + + /// The IAS SPID to use when getting a quote + #[clap(long, env = "MC_IAS_SPID")] + pub ias_spid: ProviderId, + /// gRPC listening URI for client requests. #[clap(long, env = "MC_CLIENT_LISTEN_URI")] pub client_listen_uri: FogLedgerUri, diff --git a/fog/ledger/server/src/router_server.rs b/fog/ledger/server/src/router_server.rs index 98af9eb8f0..45da1e4c7e 100644 --- a/fog/ledger/server/src/router_server.rs +++ b/fog/ledger/server/src/router_server.rs @@ -6,6 +6,7 @@ use std::{ }; use futures::executor::block_on; +use mc_attest_net::RaClient; use mc_common::{ logger::{log, Logger}, time::SystemTimeProvider, @@ -14,6 +15,7 @@ use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::{ConnectionUri, FogLedgerUri, KeyImageStoreUri}; use mc_ledger_db::LedgerDB; +use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::{ AnonymousAuthenticator, Authenticator, ConnectionUriGrpcioServer, ReadinessIndicator, TokenAuthenticator, @@ -22,27 +24,40 @@ use mc_util_uri::AdminUri; use mc_watcher::watcher_db::WatcherDB; use crate::{ - config::LedgerRouterConfig, router_admin_service::LedgerRouterAdminService, + config::LedgerRouterConfig, counters, router_admin_service::LedgerRouterAdminService, router_service::LedgerRouterService, BlockService, MerkleProofService, UntrustedTxOutService, }; -pub struct LedgerRouterServer { +pub struct LedgerRouterServer +where + E: LedgerEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ router_server: grpcio::Server, admin_server: grpcio::Server, client_listen_uri: FogLedgerUri, admin_listen_uri: AdminUri, + config: LedgerRouterConfig, + enclave: E, + ra_client: RC, + report_cache_thread: Option, logger: Logger, } -impl LedgerRouterServer { - pub fn new( +impl LedgerRouterServer +where + E: LedgerEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ + pub fn new( config: LedgerRouterConfig, enclave: E, + ra_client: RC, shards: Arc>>>, ledger: LedgerDB, watcher: WatcherDB, logger: Logger, - ) -> LedgerRouterServer + ) -> LedgerRouterServer where E: LedgerEnclaveProxy, { @@ -92,7 +107,7 @@ impl LedgerRouterServer { ledger_grpc::create_fog_merkle_proof_api(MerkleProofService::new( config.chain_id.clone(), ledger.clone(), - enclave, + enclave.clone(), client_authenticator.clone(), logger.clone(), )); @@ -137,14 +152,29 @@ impl LedgerRouterServer { Self { router_server, admin_server, - client_listen_uri: config.client_listen_uri, - admin_listen_uri: config.admin_listen_uri, + client_listen_uri: config.client_listen_uri.clone(), + admin_listen_uri: config.admin_listen_uri.clone(), + config, + enclave, + ra_client, + report_cache_thread: None, logger, } } /// Starts the server pub fn start(&mut self) { + self.report_cache_thread = Some( + ReportCacheThread::start( + self.enclave.clone(), + self.ra_client.clone(), + self.config.ias_spid, + &counters::ENCLAVE_REPORT_TIMESTAMP, + self.logger.clone(), + ) + .expect("failed starting report cache thread"), + ); + self.router_server.start(); log::info!( self.logger, @@ -167,7 +197,11 @@ impl LedgerRouterServer { } } -impl Drop for LedgerRouterServer { +impl Drop for LedgerRouterServer +where + E: LedgerEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ fn drop(&mut self) { self.stop(); } diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs new file mode 100644 index 0000000000..74bc5e657e --- /dev/null +++ b/fog/ledger/server/tests/router_connection.rs @@ -0,0 +1,846 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +//! Integration tests at the level of the fog ledger connection / fog ledger +//! grpc API + +use mc_account_keys::{AccountKey, PublicAddress}; +use mc_api::watcher::TimestampResultCode; +use mc_attest_net::{Client as AttestClient, RaClient}; +use mc_attest_verifier::{MrSignerVerifier, Verifier, DEBUG_ENCLAVE}; +use mc_blockchain_types::{BlockSignature, BlockVersion}; +use mc_common::{ + logger::{test_with_logger, Logger}, + ResponderId, +}; +use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; +use mc_fog_api::ledger::TxOutResultCode; +use mc_fog_ledger_connection::{ + Error, FogKeyImageGrpcClient, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, + KeyImageResultExtension, OutputResultExtension, +}; +use mc_fog_ledger_enclave::LedgerSgxEnclave; +use mc_fog_ledger_server::{LedgerRouterConfig, LedgerRouterServer}; +use mc_fog_test_infra::get_enclave_path; +use mc_fog_uri::{ConnectionUri, FogLedgerUri}; +use mc_ledger_db::{test_utils::recreate_ledger_db, Ledger, LedgerDB}; +use mc_transaction_core::{ + membership_proofs::compute_implied_merkle_root, ring_signature::KeyImage, tokens::Mob, Amount, + Token, +}; +use mc_util_from_random::FromRandom; +use mc_util_grpc::{GrpcRetryConfig, CHAIN_ID_MISMATCH_ERR_MSG}; +use mc_util_test_helper::{CryptoRng, RngCore, RngType, SeedableRng}; +use mc_util_uri::AdminUri; +use mc_watcher::watcher_db::WatcherDB; +use std::{ + collections::HashMap, + path::PathBuf, + str::FromStr, + sync::{Arc, RwLock}, + thread::sleep, + time::Duration, +}; +use tempdir::TempDir; +use url::Url; + +const TEST_URL: &str = "http://www.my_url1.com"; + +const OMAP_CAPACITY: u64 = 128 * 128; + +const GRPC_RETRY_CONFIG: GrpcRetryConfig = GrpcRetryConfig { + grpc_retry_count: 3, + grpc_retry_millis: 20, +}; + +fn setup_watcher_db(logger: Logger) -> (WatcherDB, PathBuf) { + let url = Url::parse(TEST_URL).unwrap(); + + let db_tmp = TempDir::new("wallet_db").expect("Could not make tempdir for wallet db"); + WatcherDB::create(db_tmp.path()).unwrap(); + let watcher = WatcherDB::open_rw(db_tmp.path(), &[url], logger).unwrap(); + let watcher_dir = db_tmp.path().to_path_buf(); + (watcher, watcher_dir) +} + +// Test that a fog ledger connection is able to get valid merkle proofs by +// hitting a fog ledger server +#[test_with_logger] +fn fog_ledger_merkle_proofs_test(logger: Logger) { + let base_port = 3230; + + let mut rng = RngType::from_seed([0u8; 32]); + + for block_version in BlockVersion::iterator() { + let alice = AccountKey::random_with_fog(&mut rng); + let bob = AccountKey::random_with_fog(&mut rng); + let charlie = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![ + alice.default_subaddress(), + bob.default_subaddress(), + charlie.default_subaddress(), + ]; + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[KeyImage::from(1)], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[KeyImage::from(2)], + &mut rng, + &mut watcher, + ); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + base_port + 7 + )) + .unwrap(); + let admin_listen_uri = + AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri: admin_listen_uri.clone(), + client_listen_uri: client_listen_uri.clone(), + client_responder_id: ResponderId::from_str(&client_listen_uri.addr()).unwrap(), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make ledger enclave client + let mut mr_signer_verifier = + MrSignerVerifier::from(mc_fog_ledger_enclave_measurement::sigstruct()); + mr_signer_verifier.allow_hardening_advisories( + mc_fog_ledger_enclave_measurement::HARDENING_ADVISORIES, + ); + + let mut verifier = Verifier::default(); + verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); + + let mut client = FogMerkleProofGrpcClient::new( + "local".to_string(), + client_listen_uri.clone(), + GRPC_RETRY_CONFIG, + verifier.clone(), + grpc_env.clone(), + logger.clone(), + ); + + // Get merkle root of num_blocks - 1 + let merkle_root = { + let temp = ledger.get_tx_out_proof_of_memberships(&[0u64]).unwrap(); + let merkle_proof = &temp[0]; + compute_implied_merkle_root(merkle_proof).unwrap() + }; + + // Get some tx outs and merkle proofs + let response = client + .get_outputs( + vec![0u64, 1u64, 2u64, 3u64, 4u64, 5u64, 6u64, 7u64, 8u64], + num_blocks - 1, + ) + .expect("get outputs failed"); + + // Test the basic fields + assert_eq!(response.num_blocks, num_blocks); + assert_eq!(response.global_txo_count, ledger.num_txos().unwrap()); + + // Validate merkle proofs + for res in response.results.iter() { + let (tx_out, proof) = res.status().unwrap().unwrap(); + let result = mc_transaction_core::membership_proofs::is_membership_proof_valid( + &tx_out, + &proof, + merkle_root.hash.as_ref(), + ) + .expect("membership proof structure failed!"); + assert!(result, "membership proof was invalid! idx = {}, output = {:?}, proof = {:?}, merkle_root = {:?}", res.index, tx_out, proof, merkle_root); + } + + // Make some queries that are out of bounds + let response = client + .get_outputs(vec![1u64, 6u64, 9u64, 14u64], num_blocks - 1) + .expect("get outputs failed"); + + // Test the basic fields + assert_eq!(response.num_blocks, num_blocks); + assert_eq!(response.global_txo_count, ledger.num_txos().unwrap()); + assert_eq!(response.results.len(), 4); + assert!(response.results[0].status().as_ref().unwrap().is_some()); + assert!(response.results[1].status().as_ref().unwrap().is_some()); + assert!(response.results[2].status().as_ref().unwrap().is_none()); + assert!(response.results[3].status().as_ref().unwrap().is_none()); + + // Check that wrong chain id results in an error + let mut client = FogMerkleProofGrpcClient::new( + "wrong".to_string(), + client_listen_uri, + GRPC_RETRY_CONFIG, + verifier, + grpc_env, + logger.clone(), + ); + + let result = client.get_outputs( + vec![0u64, 1u64, 2u64, 3u64, 4u64, 5u64, 6u64, 7u64, 8u64], + num_blocks - 1, + ); + + if let Err(err) = result { + match err { + Error::Connection( + _, + retry::Error { + error: + mc_fog_enclave_connection::Error::Rpc(grpcio::Error::RpcFailure(status)), + .. + }, + ) => { + let expected = format!("{} '{}'", CHAIN_ID_MISMATCH_ERR_MSG, "local"); + assert_eq!(status.message(), expected); + } + _ => { + panic!("unexpected grpcio error: {}", err); + } + } + } else { + panic!("Expected an error when chain-id is wrong"); + } + } + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); + } +} + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger server +#[test_with_logger] +fn fog_ledger_key_images_test(logger: Logger) { + let base_port = 3240; + + let mut rng = RngType::from_seed([0u8; 32]); + + for block_version in BlockVersion::iterator() { + let alice = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + let keys: Vec = (0..20).map(|x| KeyImage::from(x as u64)).collect(); + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + // Make WatcherDB + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[0..2], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[3..6], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[6..9], + &mut rng, + &mut watcher, + ); + + // Populate watcher with Signature and Timestamp for block 1 + let url1 = Url::parse(TEST_URL).unwrap(); + let block1 = ledger.get_block(1).unwrap(); + let signing_key_a = Ed25519Pair::from_random(&mut rng); + let filename = String::from("00/00"); + let mut signed_block_a1 = + BlockSignature::from_block_and_keypair(&block1, &signing_key_a).unwrap(); + signed_block_a1.set_signed_at(1593798844); + watcher + .add_block_signature(&url1, 1, signed_block_a1, filename.clone()) + .unwrap(); + + // Update last synced to block 2, to indicate that this URL did not participate + // in consensus for block 2. + watcher.update_last_synced(&url1, 2).unwrap(); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + base_port + 7 + )) + .unwrap(); + let admin_listen_uri = + AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri: admin_listen_uri.clone(), + client_listen_uri: client_listen_uri.clone(), + client_responder_id: ResponderId::from_str(&client_listen_uri.addr()).unwrap(), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make ledger enclave client + let mut mr_signer_verifier = + MrSignerVerifier::from(mc_fog_ledger_enclave_measurement::sigstruct()); + mr_signer_verifier.allow_hardening_advisories( + mc_fog_ledger_enclave_measurement::HARDENING_ADVISORIES, + ); + + let mut verifier = Verifier::default(); + verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); + + let mut client = FogKeyImageGrpcClient::new( + String::default(), + client_listen_uri, + GRPC_RETRY_CONFIG, + verifier, + grpc_env, + logger.clone(), + ); + + // Check on key images + let mut response = client + .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) + .expect("check_key_images failed"); + + let mut n = 1; + // adding a delay to give fog ledger time to fully initialize + while response.num_blocks != num_blocks { + response = client + .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) + .expect("check_key_images failed"); + + sleep(Duration::from_secs(10)); + // panic on the 20th time + n += 1; // + if n > 20 { + panic!("Fog ledger not fully initialized"); + } + } + + // FIXME assert_eq!(response.num_txos, ...); + assert_eq!(response.results[0].key_image, keys[0]); + assert_eq!(response.results[0].status(), Ok(Some(1))); + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[0].timestamp, 1); + + assert_eq!(response.results[1].key_image, keys[1]); + assert_eq!(response.results[1].status(), Ok(Some(1))); + assert_eq!( + response.results[1].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[1].timestamp, 1); + + // Check a key_image for a block which will never have signatures & timestamps + assert_eq!(response.results[2].key_image, keys[3]); + assert_eq!(response.results[2].status(), Ok(Some(2))); // Spent in block 2 + assert_eq!( + response.results[2].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[2].timestamp, 2); + + // Watcher has only synced 1 block, so timestamp should be behind + assert_eq!(response.results[3].key_image, keys[7]); + assert_eq!(response.results[3].status(), Ok(Some(3))); // Spent in block 3 + assert_eq!( + response.results[3].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[3].timestamp, 3); + + // Check a key_image that has not been spent + assert_eq!(response.results[4].key_image, keys[19]); + assert_eq!(response.results[4].status(), Ok(None)); // Not spent + assert_eq!( + response.results[4].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[4].timestamp, u64::MAX); + } + + // FIXME: Check a key_image that generates a DatabaseError - tough to generate + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); + } +} + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger server +#[test_with_logger] +fn fog_ledger_blocks_api_test(logger: Logger) { + let base_port = 3250; + + let mut rng = RngType::from_seed([0u8; 32]); + + let alice = AccountKey::random_with_fog(&mut rng); + let bob = AccountKey::random_with_fog(&mut rng); + let charlie = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress()], + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress(), bob.default_subaddress()], + &[KeyImage::from(1)], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[ + alice.default_subaddress(), + bob.default_subaddress(), + charlie.default_subaddress(), + ], + &[KeyImage::from(2)], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &recipients, + &[KeyImage::from(3)], + &mut rng, + &mut watcher, + ); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + base_port + 7 + )) + .unwrap(); + let admin_listen_uri = + AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri, + client_listen_uri: client_listen_uri.clone(), + client_responder_id: ResponderId::from_str(&client_listen_uri.addr()).unwrap(), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make unattested ledger client + let client = FogUntrustedLedgerGrpcClient::new( + client_listen_uri, + GRPC_RETRY_CONFIG, + grpc_env, + logger.clone(), + ); + + // Try to get a block + let queries = [0..1]; + let result = client.get_blocks(&queries).unwrap(); + // Check that we got 1 block, as expected + assert_eq!(result.blocks.len(), 1); + assert_eq!(result.blocks[0].index, 0); + assert_eq!(result.blocks[0].outputs.len(), 1); + assert_eq!(result.blocks[0].global_txo_count, 1); + assert_eq!( + result.blocks[0].timestamp_result_code, + TimestampResultCode::BlockIndexOutOfBounds as u32 + ); + assert_eq!(result.num_blocks, num_blocks); + assert_eq!(result.global_txo_count, ledger.num_txos().unwrap()); + + // Try to get two blocks + let queries = [1..3]; + let result = client.get_blocks(&queries).unwrap(); + + // Check that we got 2 blocks, as expected + assert_eq!(result.blocks.len(), 2); + assert_eq!(result.blocks[0].index, 1); + assert_eq!(result.blocks[0].outputs.len(), 2); + assert_eq!(result.blocks[0].global_txo_count, 3); + assert_eq!( + result.blocks[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(result.blocks[1].index, 2); + assert_eq!(result.blocks[1].outputs.len(), 3); + assert_eq!(result.blocks[1].global_txo_count, 6); + assert_eq!( + result.blocks[1].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(result.num_blocks, num_blocks); + assert_eq!(result.global_txo_count, ledger.num_txos().unwrap()); + } + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); +} + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger server +#[test_with_logger] +fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { + let base_port = 3260; + + let mut rng = RngType::from_seed([0u8; 32]); + + let alice = AccountKey::random_with_fog(&mut rng); + let bob = AccountKey::random_with_fog(&mut rng); + let charlie = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress()], + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[alice.default_subaddress(), bob.default_subaddress()], + &[KeyImage::from(1)], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &[ + alice.default_subaddress(), + bob.default_subaddress(), + charlie.default_subaddress(), + ], + &[KeyImage::from(2)], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + BlockVersion::MAX, + &mut ledger, + &recipients, + &[KeyImage::from(3)], + &mut rng, + &mut watcher, + ); + + { + // Make LedgerServer + let client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + base_port + 7 + )) + .unwrap(); + let admin_listen_uri = + AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri, + client_listen_uri: client_listen_uri.clone(), + client_responder_id: ResponderId::from_str(&client_listen_uri.addr()).unwrap(), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let ra_client = + AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut ledger_server = LedgerRouterServer::new( + config, + enclave, + ra_client, + Arc::new(RwLock::new(HashMap::new())), + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + ledger_server.start(); + + // Make unattested ledger client + let client = FogUntrustedLedgerGrpcClient::new( + client_listen_uri, + GRPC_RETRY_CONFIG, + grpc_env, + logger.clone(), + ); + + // Get a tx_out that is actually in the ledger + let real_tx_out0 = { ledger.get_tx_out_by_index(0).unwrap() }; + + // Try to get tx out records + let queries: Vec = + vec![(&[0u8; 32]).into(), real_tx_out0.public_key]; + let result = client.get_tx_outs(queries).unwrap(); + // Check that we got expected num_blocks value + assert_eq!(result.num_blocks, 4); + // Check that we got 2 results, as expected + assert_eq!(result.results.len(), 2); + assert_eq!( + &result.results[0].tx_out_pubkey.clone().unwrap().data[..], + &[0u8; 32] + ); + assert_eq!(result.results[0].result_code, TxOutResultCode::NotFound); + assert_eq!( + &result.results[1].tx_out_pubkey.clone().unwrap().data[..], + &real_tx_out0.public_key.as_bytes()[..] + ); + assert_eq!(result.results[1].result_code, TxOutResultCode::Found); + assert_eq!(result.results[1].tx_out_global_index, 0); + assert_eq!(result.results[1].block_index, 0); + assert_eq!( + result.results[1].timestamp_result_code, + TimestampResultCode::BlockIndexOutOfBounds as u32 + ); + } + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); +} + +// Infra + +/// Adds a block containing one txo for each provided recipient and returns new +/// block height. +/// +/// # Arguments +/// * `ledger_db` +/// * `recipients` - Recipients of outputs. +/// * `rng` +fn add_block_to_ledger( + block_version: BlockVersion, + ledger_db: &mut LedgerDB, + recipients: &[PublicAddress], + key_images: &[KeyImage], + rng: &mut (impl CryptoRng + RngCore), + watcher: &mut WatcherDB, +) -> u64 { + let amount = Amount::new(10, Mob::ID); + let block_data = mc_ledger_db::test_utils::add_block_to_ledger( + ledger_db, + block_version, + recipients, + amount, + key_images, + rng, + ) + .expect("failed to add block"); + let block_index = block_data.block().index; + + let signature = block_data.signature().expect("missing signature"); + for src_url in watcher.get_config_urls().unwrap().iter() { + watcher + .add_block_signature( + src_url, + block_index, + signature.clone(), + format!("00/{}", block_index), + ) + .expect("Could not add block signature"); + } + + block_index + 1 +} From 90f99c79245021a1d941332e35676698eda415ef Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Wed, 15 Feb 2023 10:41:31 -0800 Subject: [PATCH 02/11] Port key image test in router_connection.rs to streaming API --- Cargo.lock | 8 + fog/ledger/connection/Cargo.toml | 8 + fog/ledger/connection/src/lib.rs | 3 + fog/ledger/connection/src/router_client.rs | 265 ++++++++++++++++++ fog/ledger/enclave/impl/src/lib.rs | 2 +- fog/ledger/server/src/bin/key_image_store.rs | 4 + .../server/src/key_image_router_server.rs | 117 -------- .../server/src/key_image_router_service.rs | 86 ------ fog/ledger/server/src/key_image_service.rs | 37 ++- .../server/src/key_image_store_server.rs | 50 +++- fog/ledger/server/src/router_handlers.rs | 2 +- fog/ledger/server/tests/router_connection.rs | 115 ++++++-- fog/ledger/server/tests/store_tests.rs | 4 + 13 files changed, 454 insertions(+), 247 deletions(-) create mode 100644 fog/ledger/connection/src/router_client.rs delete mode 100644 fog/ledger/server/src/key_image_router_server.rs delete mode 100644 fog/ledger/server/src/key_image_router_service.rs diff --git a/Cargo.lock b/Cargo.lock index af7e8c1417..4b67d5b5d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3834,22 +3834,30 @@ dependencies = [ name = "mc-fog-ledger-connection" version = "4.0.0" dependencies = [ + "aes-gcm", "displaydoc", + "futures", "grpcio", "mc-api", + "mc-attest-ake", + "mc-attest-core", "mc-attest-verifier", "mc-blockchain-types", "mc-common", "mc-crypto-keys", + "mc-crypto-noise", + "mc-crypto-rand", "mc-fog-api", "mc-fog-enclave-connection", "mc-fog-types", "mc-fog-uri", "mc-transaction-core", "mc-util-grpc", + "mc-util-serial", "mc-util-uri", "protobuf", "retry", + "sha2 0.10.6", ] [[package]] diff --git a/fog/ledger/connection/Cargo.toml b/fog/ledger/connection/Cargo.toml index 63da489090..790612b5ac 100644 --- a/fog/ledger/connection/Cargo.toml +++ b/fog/ledger/connection/Cargo.toml @@ -8,12 +8,17 @@ license = "GPL-3.0" [dependencies] # mobilecoin mc-api = { path = "../../../api" } +mc-attest-ake = { path = "../../../attest/ake" } +mc-attest-core = { path = "../../../attest/core" } mc-attest-verifier = { path = "../../../attest/verifier" } mc-blockchain-types = { path = "../../../blockchain/types" } mc-common = { path = "../../../common", features = ["log"] } mc-crypto-keys = { path = "../../../crypto/keys" } +mc-crypto-noise = { path = "../../../crypto/noise" } +mc-crypto-rand = { path = "../../../crypto/rand" } mc-transaction-core = { path = "../../../transaction/core" } mc-util-grpc = { path = "../../../util/grpc" } +mc-util-serial = { path = "../../../util/serial" } mc-util-uri = { path = "../../../util/uri" } # fog @@ -23,10 +28,13 @@ mc-fog-types = { path = "../../types" } mc-fog-uri = { path = "../../uri" } # third-party +aes-gcm = "0.10.1" displaydoc = { version = "0.2", default-features = false } +futures = "0.3" grpcio = "0.12.0" protobuf = "2.27.1" retry = "2.0" +sha2 = { version = "0.10", default-features = false } [dev-dependencies] mc-common = { path = "../../../common", features = ["loggers"] } diff --git a/fog/ledger/connection/src/lib.rs b/fog/ledger/connection/src/lib.rs index 5278ab196d..7ca5c5a2a5 100644 --- a/fog/ledger/connection/src/lib.rs +++ b/fog/ledger/connection/src/lib.rs @@ -17,3 +17,6 @@ pub use merkle_proof::{FogMerkleProofGrpcClient, OutputError, OutputResultExtens mod untrusted; pub use untrusted::FogUntrustedLedgerGrpcClient; + +mod router_client; +pub use router_client::LedgerGrpcClient; diff --git a/fog/ledger/connection/src/router_client.rs b/fog/ledger/connection/src/router_client.rs new file mode 100644 index 0000000000..adc33c30bb --- /dev/null +++ b/fog/ledger/connection/src/router_client.rs @@ -0,0 +1,265 @@ +use aes_gcm::Aes256Gcm; +use futures::{executor::block_on, SinkExt, TryStreamExt}; +use grpcio::{ChannelBuilder, ClientDuplexReceiver, ClientDuplexSender, Environment}; +use mc_attest_ake::{ + AuthResponseInput, ClientInitiate, Error as AttestAkeError, Ready, Start, Transition, +}; +use mc_attest_core::VerificationReport; +use mc_attest_verifier::Verifier; +use mc_common::logger::{log, o, Logger}; +use mc_crypto_keys::X25519; +use mc_crypto_noise::CipherError; +use mc_crypto_rand::McRng; +use mc_fog_api::{ + attest::{AuthMessage, Message}, + ledger::{LedgerRequest, LedgerResponse}, + ledger_grpc::LedgerApiClient, +}; +use mc_fog_types::ledger::{CheckKeyImagesRequest, CheckKeyImagesResponse, KeyImageQuery}; +use mc_fog_uri::FogLedgerUri; +use mc_transaction_core::ring_signature::KeyImage; +use mc_util_grpc::ConnectionUriGrpcioChannel; +use mc_util_serial::DecodeError; +use mc_util_uri::{ConnectionUri, UriConversionError}; +use sha2::Sha512; +use std::sync::Arc; + +/// A high-level object mediating requests to the fog ledger router service +pub struct LedgerGrpcClient { + /// A logger object + logger: Logger, + + /// The URI of the router to communicate with + uri: FogLedgerUri, + + /// An object which can verify a fog node's provided IAS report + verifier: Verifier, + + /// The AKE state machine object, if one is available. + attest_cipher: Option>, + + /// Sends requests to the fog view router + request_sender: ClientDuplexSender, + + /// Receives responses from the fog view router + response_receiver: ClientDuplexReceiver, + + /// Low-lever ledger API client + _client: LedgerApiClient, +} + +impl LedgerGrpcClient { + /// Creates a new fog ledger router grpc client and opens a streaming + /// connection to the fog ledger router service. + /// + /// Arguments: + /// * uri: The Uri to connect to + /// * verifier: The attestation verifier + /// * env: A grpc environment (thread pool) to use for this connection + /// * logger: For logging + pub fn new( + uri: FogLedgerUri, + verifier: Verifier, + env: Arc, + logger: Logger, + ) -> Self { + let logger = logger.new(o!("mc.fog.ledger.router.uri" => uri.to_string())); + + let ch = ChannelBuilder::default_channel_builder(env).connect_to_uri(&uri, &logger); + let client = LedgerApiClient::new(ch); + let (request_sender, response_receiver) = client + .request() + .expect("Could not retrieve grpc sender and receiver."); + + Self { + logger, + attest_cipher: None, + _client: client, + request_sender, + response_receiver, + uri, + verifier, + } + } + + fn is_attested(&self) -> bool { + self.attest_cipher.is_some() + } + + async fn attest(&mut self) -> Result { + // If we have an existing attestation, nuke it. + self.deattest(); + + let mut csprng = McRng::default(); + + let initiator = Start::new(self.uri.responder_id()?.to_string()); + + let init_input = ClientInitiate::::default(); + let (initiator, auth_request_output) = initiator.try_next(&mut csprng, init_input)?; + + let attested_message: AuthMessage = auth_request_output.into(); + let mut request = LedgerRequest::new(); + request.set_auth(attested_message); + self.request_sender + .send((request.clone(), grpcio::WriteFlags::default())) + .await?; + + let mut response = self + .response_receiver + .try_next() + .await? + .ok_or(Error::ResponseNotReceived)?; + let auth_response_msg = response.take_auth(); + + // Process server response, check if key exchange is successful + let auth_response_event = + AuthResponseInput::new(auth_response_msg.into(), self.verifier.clone()); + let (initiator, verification_report) = + initiator.try_next(&mut csprng, auth_response_event)?; + + self.attest_cipher = Some(initiator); + + Ok(verification_report) + } + + fn deattest(&mut self) { + if self.is_attested() { + log::trace!(self.logger, "Tearing down existing attested connection."); + self.attest_cipher = None; + } + } + + /// Check one or more key images against the ledger router service + pub async fn check_key_images( + &mut self, + key_images: &[KeyImage], + ) -> Result { + log::trace!(self.logger, "Check key images was called"); + if !self.is_attested() { + log::warn!(self.logger, "attesting now"); + let verification_report = self.attest().await; + verification_report?; + } + + log::warn!(self.logger, "attested"); + let key_images_queries = key_images + .iter() + .map(|&key_image| KeyImageQuery { + key_image, + start_block: 0, + }) + .collect(); + log::warn!(self.logger, "queries are: {:?}", key_images_queries); + let key_images_request = CheckKeyImagesRequest { + queries: key_images_queries, + }; + + let aad = vec![]; + + let msg = { + let attest_cipher = self + .attest_cipher + .as_mut() + .expect("no enclave_connection even though attest succeeded"); + + let mut msg = Message::new(); + msg.set_channel_id(Vec::from(attest_cipher.binding())); + msg.set_aad(aad.clone()); + + let plaintext_bytes = mc_util_serial::encode(&key_images_request); + + log::warn!(self.logger, "about to encrypt"); + let request_ciphertext = attest_cipher.encrypt(&aad, &plaintext_bytes)?; + log::warn!(self.logger, "encrypted"); + msg.set_data(request_ciphertext); + msg + }; + let mut request = LedgerRequest::new(); + request.set_check_key_images(msg); + + self.request_sender + .send((request.clone(), grpcio::WriteFlags::default())) + .await?; + + log::warn!(self.logger, "about to receive"); + let message = self + .response_receiver + .try_next() + .await? + .ok_or(Error::ResponseNotReceived)? + .take_check_key_image_response(); + log::warn!(self.logger, "received"); + + { + let attest_cipher = self + .attest_cipher + .as_mut() + .expect("no enclave_connection even though attest succeeded"); + + log::warn!(self.logger, "about to decrypt"); + let plaintext_bytes = attest_cipher.decrypt(message.get_aad(), message.get_data())?; + log::warn!(self.logger, "about to decode"); + let plaintext_response: CheckKeyImagesResponse = + mc_util_serial::decode(&plaintext_bytes)?; + Ok(plaintext_response) + } + } +} + +impl Drop for LedgerGrpcClient { + fn drop(&mut self) { + block_on(self.request_sender.close()).expect("Couldn't close the router request sender"); + } +} + +/// Errors related to the Fog View Router Client. +#[derive(Debug)] +pub enum Error { + /// Decode errors. + Decode(DecodeError), + + /// Uri conversion errors. + UriConversion(UriConversionError), + + /// Cipher errors. + Cipher(CipherError), + + /// Attestation errors. + Attestation(AttestAkeError), + + /// Grpc errors. + Grpc(grpcio::Error), + + /// Response not received + ResponseNotReceived, +} + +impl From for Error { + fn from(err: DecodeError) -> Self { + Self::Decode(err) + } +} + +impl From for Error { + fn from(err: CipherError) -> Self { + Self::Cipher(err) + } +} + +impl From for Error { + fn from(err: grpcio::Error) -> Self { + Self::Grpc(err) + } +} + +impl From for Error { + fn from(err: UriConversionError) -> Self { + Self::UriConversion(err) + } +} + +impl From for Error { + fn from(err: AttestAkeError) -> Self { + Self::Attestation(err) + } +} diff --git a/fog/ledger/enclave/impl/src/lib.rs b/fog/ledger/enclave/impl/src/lib.rs index 7955475ea9..8bd870d0a1 100644 --- a/fog/ledger/enclave/impl/src/lib.rs +++ b/fog/ledger/enclave/impl/src/lib.rs @@ -354,7 +354,7 @@ where &self, auth_request: NonceAuthRequest, ) -> Result<(NonceAuthResponse, NonceSession)> { - self.ake.frontend_accept(auth_request).map_err(|e| e.into()) + Ok(self.ake.frontend_accept(auth_request)?) } } diff --git a/fog/ledger/server/src/bin/key_image_store.rs b/fog/ledger/server/src/bin/key_image_store.rs index e55b7ff7ef..c893ab64a0 100644 --- a/fog/ledger/server/src/bin/key_image_store.rs +++ b/fog/ledger/server/src/bin/key_image_store.rs @@ -2,6 +2,7 @@ use clap::Parser; use grpcio::{RpcStatus, RpcStatusCode}; +use mc_attest_net::{Client, RaClient}; use mc_common::{logger::log, time::SystemTimeProvider}; use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; use mc_fog_ledger_server::{KeyImageStoreServer, LedgerStoreConfig, ShardingStrategy}; @@ -40,10 +41,13 @@ fn main() { let watcher = WatcherDB::open_ro(&config.watcher_db, logger.clone()).expect("Could not open watcher DB"); + let ias_client = Client::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut store_server = match config.sharding_strategy.clone() { ShardingStrategy::Epoch(sharding_strategy) => KeyImageStoreServer::new_from_config( config.clone(), enclave, + ias_client, db, watcher, sharding_strategy, diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs deleted file mode 100644 index bce08d6dd8..0000000000 --- a/fog/ledger/server/src/key_image_router_server.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use futures::executor::block_on; -use mc_common::logger::{log, Logger}; -use mc_fog_api::ledger_grpc; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; -use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; - -use crate::{ - config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService, - router_admin_service::LedgerRouterAdminService, -}; - -pub struct KeyImageRouterServer { - router_server: grpcio::Server, - admin_server: grpcio::Server, - logger: Logger, -} - -impl KeyImageRouterServer { - pub fn new( - config: LedgerRouterConfig, - enclave: E, - shards: Arc>>>, - logger: Logger, - ) -> KeyImageRouterServer - where - E: LedgerEnclaveProxy, - { - let readiness_indicator = ReadinessIndicator::default(); - - let env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("key-image-router-server".to_string()) - .build(), - ); - - // Health check service - will be used in both cases - let health_service = - mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) - .into_service(); - - // Build our router server. - // Init ledger router service. - let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( - enclave, - shards.clone(), - logger.clone(), - )); - log::debug!(logger, "Constructed Key Image Router GRPC Service"); - - // Init ledger router admin service. - let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( - LedgerRouterAdminService::new(shards, logger.clone()), - ); - log::debug!(logger, "Constructed Key Image Router Admin GRPC Service"); - - // Package service into grpc server - log::info!( - logger, - "Starting Key Image Router server on {}", - config.client_listen_uri.addr(), - ); - - let router_server_builder = grpcio::ServerBuilder::new(env.clone()) - .register_service(ledger_router_service) - .register_service(health_service) - .bind_using_uri(&config.client_listen_uri, logger.clone()); - let admin_server_builder = grpcio::ServerBuilder::new(env) - .register_service(ledger_router_admin_service) - .bind_using_uri(&config.admin_listen_uri, logger.clone()); - - let router_server = router_server_builder.build().unwrap(); - let admin_server = admin_server_builder.build().unwrap(); - - Self { - router_server, - admin_server, - logger, - } - } - - /// Starts the server - pub fn start(&mut self) { - self.router_server.start(); - for (host, port) in self.router_server.bind_addrs() { - log::info!(self.logger, "Router API listening on {}:{}", host, port); - } - self.admin_server.start(); - for (host, port) in self.admin_server.bind_addrs() { - log::info!( - self.logger, - "Router Admin API listening on {}:{}", - host, - port - ); - } - } - - /// Stops the server - pub fn stop(&mut self) { - block_on(self.router_server.shutdown()).expect("Could not stop router grpc server"); - block_on(self.admin_server.shutdown()).expect("Could not stop router admin grpc server"); - } -} - -impl Drop for KeyImageRouterServer { - fn drop(&mut self) { - self.stop(); - } -} diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs deleted file mode 100644 index 707674547a..0000000000 --- a/fog/ledger/server/src/key_image_router_service.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use futures::{FutureExt, TryFutureExt}; -use grpcio::{DuplexSink, RequestStream, RpcContext}; -use mc_common::logger::{log, Logger}; -use mc_fog_api::{ - ledger::{LedgerRequest, LedgerResponse}, - ledger_grpc::{self, LedgerApi}, -}; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::KeyImageStoreUri; -use mc_util_grpc::rpc_logger; -use mc_util_metrics::SVC_COUNTERS; - -use crate::router_handlers; - -#[derive(Clone)] -pub struct KeyImageRouterService -where - E: LedgerEnclaveProxy, -{ - enclave: E, - shards: Arc>>>, - query_retries: usize, - logger: Logger, -} - -impl KeyImageRouterService { - /// Creates a new LedgerRouterService that can be used by a gRPC server to - /// fulfill gRPC requests. - #[allow(dead_code)] // FIXME - pub fn new( - enclave: E, - shards: Arc>>>, - query_retries: usize, - logger: Logger, - ) -> Self { - Self { - enclave, - shards, - query_retries, - logger, - } - } -} - -impl LedgerApi for KeyImageRouterService -where - E: LedgerEnclaveProxy, -{ - fn request( - &mut self, - ctx: RpcContext, - requests: RequestStream, - responses: DuplexSink, - ) { - let _timer = SVC_COUNTERS.req(&ctx); - mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { - log::warn!( - self.logger, - "Streaming GRPC Ledger API only partially implemented." - ); - let logger = logger.clone(); - - let shards = self.shards.read().expect("RwLock poisoned"); - let future = router_handlers::handle_requests( - shards.values().cloned().collect(), - self.enclave.clone(), - requests, - responses, - self.query_retries, - logger.clone(), - ) - .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) - // TODO: Do more with the error than just push it to the log. - .map(|_| ()); - - ctx.spawn(future) - }); - } -} diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index a23deaa957..5953a44ff6 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -73,7 +73,36 @@ impl KeyImageService { self.db_poll_shared_state.clone() } - pub fn auth_impl( + pub fn auth_store( + &mut self, + mut req: AuthMessage, + logger: &Logger, + ) -> Result { + // TODO: Use the prost message directly, once available + match self.enclave.frontend_accept(req.take_data().into()) { + Ok((response, _)) => { + let mut result = attest::AuthMessage::new(); + result.set_data(response.into()); + Ok(result) + } + Err(client_error) => { + // There's no requirement on the remote party to trigger this, so it's debug. + log::debug!( + logger, + "KeyImageStoreApi::frontend_accept failed: {}", + client_error + ); + let rpc_permissions_error = rpc_permissions_error( + "client_auth", + format!("Permission denied: {}", client_error), + logger, + ); + Err(rpc_permissions_error) + } + } + } + + pub fn auth_service( &mut self, mut req: AuthMessage, logger: &Logger, @@ -247,7 +276,7 @@ impl FogKeyImageApi for KeyImageServic return send_result(ctx, sink, err.into(), logger); } - match self.auth_impl(request, logger) { + match self.auth_service(request, logger) { Ok(response) => { send_result(ctx, sink, Ok(response), logger); } @@ -280,7 +309,7 @@ impl KeyImageStoreApi for KeyImageServ return send_result(ctx, sink, err.into(), logger); } - match self.auth_impl(req, logger) { + match self.auth_store(req, logger) { Ok(response) => { send_result(ctx, sink, Ok(response), logger); } @@ -289,7 +318,7 @@ impl KeyImageStoreApi for KeyImageServ // it. log::info!( logger, - "LedgerEnclave::client_accept failed: {}", + "LedgerEnclave::frontend_accept failed: {}", client_error ); // TODO: increment failed inbound peering counter. diff --git a/fog/ledger/server/src/key_image_store_server.rs b/fog/ledger/server/src/key_image_store_server.rs index d07090c139..552e251773 100644 --- a/fog/ledger/server/src/key_image_store_server.rs +++ b/fog/ledger/server/src/key_image_store_server.rs @@ -3,6 +3,8 @@ use std::sync::{Arc, Mutex}; use futures::executor::block_on; +use mc_attest_core::ProviderId; +use mc_attest_net::RaClient; use mc_common::{ logger::{log, Logger}, time::TimeProvider, @@ -11,6 +13,7 @@ use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; use mc_ledger_db::LedgerDB; +use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::{ AnonymousAuthenticator, Authenticator, ConnectionUriGrpcioServer, ReadinessIndicator, TokenAuthenticator, @@ -18,36 +21,43 @@ use mc_util_grpc::{ use mc_watcher::watcher_db::WatcherDB; use crate::{ - config::LedgerStoreConfig, db_fetcher::DbFetcher, server::DbPollSharedState, + config::LedgerStoreConfig, counters, db_fetcher::DbFetcher, server::DbPollSharedState, sharding_strategy::ShardingStrategy, KeyImageClientListenUri, KeyImageService, }; -pub struct KeyImageStoreServer +pub struct KeyImageStoreServer where E: LedgerEnclaveProxy, SS: ShardingStrategy + Send + Sync + 'static, + RC: RaClient + Send + Sync + 'static, { server: grpcio::Server, client_listen_uri: KeyImageStoreUri, db_fetcher: DbFetcher, + enclave: E, + ra_client: RC, + report_cache_thread: Option, + ias_spid: ProviderId, logger: Logger, } -impl KeyImageStoreServer +impl KeyImageStoreServer where E: LedgerEnclaveProxy, SS: ShardingStrategy + Send + Sync + 'static, + RC: RaClient + Send + Sync + 'static, { /// Creates a new key image store server instance pub fn new_from_config( config: LedgerStoreConfig, enclave: E, + ra_client: RC, ledger: LedgerDB, watcher: WatcherDB, sharding_strategy: SS, time_provider: impl TimeProvider + 'static, logger: Logger, - ) -> KeyImageStoreServer { + ) -> KeyImageStoreServer { let client_authenticator: Arc = if let Some(shared_secret) = config.client_auth_token_secret.as_ref() { Arc::new(TokenAuthenticator::new( @@ -64,6 +74,8 @@ where client_authenticator, config.client_listen_uri, enclave, + ra_client, + config.ias_spid, ledger, watcher, sharding_strategy, @@ -76,11 +88,13 @@ where client_authenticator: Arc, client_listen_uri: KeyImageStoreUri, enclave: E, + ra_client: RC, + ias_spid: ProviderId, ledger: LedgerDB, watcher: WatcherDB, sharding_strategy: SS, logger: Logger, - ) -> KeyImageStoreServer { + ) -> KeyImageStoreServer { let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); let key_image_service = KeyImageService::new( @@ -97,6 +111,8 @@ where key_image_service, client_listen_uri, enclave, + ra_client, + ias_spid, sharding_strategy, logger, ) @@ -106,9 +122,11 @@ where mut key_image_service: KeyImageService, client_listen_uri: KeyImageStoreUri, enclave: E, + ra_client: RC, + ias_spid: ProviderId, sharding_strategy: SS, logger: Logger, - ) -> KeyImageStoreServer { + ) -> KeyImageStoreServer { let readiness_indicator = ReadinessIndicator::default(); let env = Arc::new( @@ -145,7 +163,7 @@ where let db_fetcher = DbFetcher::new( key_image_service.get_ledger(), - enclave, + enclave.clone(), sharding_strategy, key_image_service.get_watcher(), key_image_service.get_db_poll_shared_state(), @@ -157,12 +175,27 @@ where server, client_listen_uri, db_fetcher, + enclave, + ra_client, + ias_spid, + report_cache_thread: None, logger, } } /// Starts the server pub fn start(&mut self) { + self.report_cache_thread = Some( + ReportCacheThread::start( + self.enclave.clone(), + self.ra_client.clone(), + self.ias_spid, + &counters::ENCLAVE_REPORT_TIMESTAMP, + self.logger.clone(), + ) + .expect("failed starting report cache thread"), + ); + self.server.start(); log::info!( self.logger, @@ -179,10 +212,11 @@ where } } -impl Drop for KeyImageStoreServer +impl Drop for KeyImageStoreServer where E: LedgerEnclaveProxy, SS: ShardingStrategy + Send + Sync + 'static, + RC: RaClient + Send + Sync + 'static, { fn drop(&mut self) { self.stop(); diff --git a/fog/ledger/server/src/router_handlers.rs b/fog/ledger/server/src/router_handlers.rs index c71df6064c..217ddcd412 100644 --- a/fog/ledger/server/src/router_handlers.rs +++ b/fog/ledger/server/src/router_handlers.rs @@ -332,7 +332,7 @@ async fn authenticate_ledger_store( ledger_store_url: KeyImageStoreUri, logger: Logger, ) -> Result<(), RouterServerError> { - let ledger_store_id = ResponderId::from_str(&ledger_store_url.to_string())?; + let ledger_store_id = ledger_store_url.responder_id()?; let client_auth_request = enclave.ledger_store_init(ledger_store_id.clone())?; let grpc_env = Arc::new( grpcio::EnvBuilder::new() diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index 74bc5e657e..853bb8ba68 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -3,6 +3,8 @@ //! Integration tests at the level of the fog ledger connection / fog ledger //! grpc API +use futures::executor::block_on; +use grpcio::ChannelBuilder; use mc_account_keys::{AccountKey, PublicAddress}; use mc_api::watcher::TimestampResultCode; use mc_attest_net::{Client as AttestClient, RaClient}; @@ -10,25 +12,29 @@ use mc_attest_verifier::{MrSignerVerifier, Verifier, DEBUG_ENCLAVE}; use mc_blockchain_types::{BlockSignature, BlockVersion}; use mc_common::{ logger::{test_with_logger, Logger}, + time::SystemTimeProvider, ResponderId, }; use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; -use mc_fog_api::ledger::TxOutResultCode; +use mc_fog_api::{ledger::TxOutResultCode, ledger_grpc::KeyImageStoreApiClient}; use mc_fog_ledger_connection::{ - Error, FogKeyImageGrpcClient, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, - KeyImageResultExtension, OutputResultExtension, + Error, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, KeyImageResultExtension, + LedgerGrpcClient, OutputResultExtension, }; use mc_fog_ledger_enclave::LedgerSgxEnclave; -use mc_fog_ledger_server::{LedgerRouterConfig, LedgerRouterServer}; +use mc_fog_ledger_server::{ + sharding_strategy::EpochShardingStrategy, KeyImageStoreServer, LedgerRouterConfig, + LedgerRouterServer, LedgerStoreConfig, ShardingStrategy, +}; use mc_fog_test_infra::get_enclave_path; -use mc_fog_uri::{ConnectionUri, FogLedgerUri}; +use mc_fog_uri::{ConnectionUri, FogLedgerUri, KeyImageStoreUri}; use mc_ledger_db::{test_utils::recreate_ledger_db, Ledger, LedgerDB}; use mc_transaction_core::{ membership_proofs::compute_implied_merkle_root, ring_signature::KeyImage, tokens::Mob, Amount, Token, }; use mc_util_from_random::FromRandom; -use mc_util_grpc::{GrpcRetryConfig, CHAIN_ID_MISMATCH_ERR_MSG}; +use mc_util_grpc::{ConnectionUriGrpcioChannel, GrpcRetryConfig, CHAIN_ID_MISMATCH_ERR_MSG}; use mc_util_test_helper::{CryptoRng, RngCore, RngType, SeedableRng}; use mc_util_uri::AdminUri; use mc_watcher::watcher_db::WatcherDB; @@ -346,7 +352,62 @@ fn fog_ledger_key_images_test(logger: Logger) { watcher.update_last_synced(&url1, 2).unwrap(); { - // Make LedgerServer + // Make Key Image Store + let store_uri = KeyImageStoreUri::from_str(&format!( + "insecure-key-image-store://127.0.0.1:{}", + base_port + 9 + )) + .unwrap(); + let store_admin_uri = + AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 10)) + .unwrap(); + let store_config = LedgerStoreConfig { + chain_id: "local".to_string(), + client_responder_id: store_uri + .responder_id() + .expect("Couldn't get responder ID for store"), + client_listen_uri: store_uri.clone(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir.clone(), + ias_api_key: Default::default(), + ias_spid: Default::default(), + admin_listen_uri: Some(store_admin_uri), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + omap_capacity: OMAP_CAPACITY, + sharding_strategy: ShardingStrategy::Epoch(EpochShardingStrategy::default()), + }; + let store_enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &store_config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + let ra_client = + AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); + let mut store_server = KeyImageStoreServer::new_from_config( + store_config, + store_enclave, + ra_client, + ledger.clone(), + watcher.clone(), + EpochShardingStrategy::default(), + SystemTimeProvider::default(), + logger.clone(), + ); + + // Make Key Image Store client + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let store_client = KeyImageStoreApiClient::new( + ChannelBuilder::default_channel_builder(grpc_env.clone()) + .connect_to_uri(&store_uri, &logger), + ); + let mut store_clients = HashMap::new(); + store_clients.insert(store_uri, Arc::new(store_client)); + let shards = Arc::new(RwLock::new(store_clients)); + + // Make Router Server let client_listen_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", base_port + 7 @@ -354,7 +415,7 @@ fn fog_ledger_key_images_test(logger: Logger) { .unwrap(); let admin_listen_uri = AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); - let config = LedgerRouterConfig { + let router_config = LedgerRouterConfig { chain_id: "local".to_string(), ledger_db: db_full_path.to_path_buf(), watcher_db: watcher_dir, @@ -371,26 +432,25 @@ fn fog_ledger_key_images_test(logger: Logger) { let enclave = LedgerSgxEnclave::new( get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), - &config.client_responder_id, + &router_config.client_responder_id, OMAP_CAPACITY, logger.clone(), ); - let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); - let ra_client = - AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); - let mut ledger_server = LedgerRouterServer::new( - config, + AttestClient::new(&router_config.ias_api_key).expect("Could not create IAS client"); + let mut router_server = LedgerRouterServer::new( + router_config, enclave, ra_client, - Arc::new(RwLock::new(HashMap::new())), + shards, ledger.clone(), watcher.clone(), logger.clone(), ); - ledger_server.start(); + store_server.start(); + router_server.start(); // Make ledger enclave client let mut mr_signer_verifier = @@ -402,26 +462,21 @@ fn fog_ledger_key_images_test(logger: Logger) { let mut verifier = Verifier::default(); verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); - let mut client = FogKeyImageGrpcClient::new( - String::default(), - client_listen_uri, - GRPC_RETRY_CONFIG, - verifier, - grpc_env, - logger.clone(), - ); + let mut client = + LedgerGrpcClient::new(client_listen_uri, verifier, grpc_env, logger.clone()); // Check on key images - let mut response = client - .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) - .expect("check_key_images failed"); + let mut response = + block_on(client.check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]])) + .expect("check_key_images failed"); let mut n = 1; // adding a delay to give fog ledger time to fully initialize while response.num_blocks != num_blocks { - response = client - .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) - .expect("check_key_images failed"); + response = block_on( + client.check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]), + ) + .expect("check_key_images failed"); sleep(Duration::from_secs(10)); // panic on the 20th time diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index 73bf6a42df..ce8616bd26 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -183,11 +183,15 @@ pub fn direct_key_image_store_check(logger: Logger) { Arc::new(AnonymousAuthenticator::default()), logger.clone(), ); + let ra_client = + AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); let mut store_server = KeyImageStoreServer::new_from_service( store_service, client_listen_uri, enclave.clone(), + ra_client, + store_config.ias_spid, EpochShardingStrategy::default(), logger.clone(), ); From dd0fc7173506a44b75495c1c2d58b0f33e0fe4bc Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Wed, 15 Feb 2023 11:08:33 -0800 Subject: [PATCH 03/11] Fixups from rebase --- fog/ledger/server/tests/store.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/fog/ledger/server/tests/store.rs b/fog/ledger/server/tests/store.rs index 9cc30e3027..883850c6f5 100644 --- a/fog/ledger/server/tests/store.rs +++ b/fog/ledger/server/tests/store.rs @@ -28,7 +28,6 @@ use mc_fog_ledger_server::{ use mc_fog_types::ledger::{CheckKeyImagesRequest, KeyImageQuery}; use mc_fog_uri::{ConnectionUri, KeyImageStoreScheme, KeyImageStoreUri}; use mc_ledger_db::{test_utils::recreate_ledger_db, LedgerDB}; -use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::AnonymousAuthenticator; use mc_util_metrics::{IntGauge, OpMetrics}; use mc_util_test_helper::{Rng, RngType, SeedableRng}; @@ -181,27 +180,20 @@ pub fn direct_key_image_store_check(logger: Logger) { logger.clone(), ); - let mut store_server = KeyImageStoreServer::new_from_service( - store_service, - client_listen_uri, - enclave.clone(), - EpochShardingStrategy::default(), - logger.clone(), - ); - store_server.start(); - // Set up IAS verficiation // This will be a SimClient in testing contexts. let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); - let _report_cache_thread = ReportCacheThread::start( + let mut store_server = KeyImageStoreServer::new_from_service( + store_service, + client_listen_uri, enclave.clone(), ias_client, store_config.ias_spid, - &TEST_ENCLAVE_REPORT_TIMESTAMP, + EpochShardingStrategy::default(), logger.clone(), - ) - .expect("Failed to start IAS client."); + ); + store_server.start(); // Make GRPC client for sending requests. From 74091179edf1efcbb323fff006aa1dd1f834cf52 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Wed, 15 Feb 2023 14:14:56 -0800 Subject: [PATCH 04/11] remove logging statements --- fog/ledger/connection/src/router_client.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fog/ledger/connection/src/router_client.rs b/fog/ledger/connection/src/router_client.rs index adc33c30bb..d31d75421c 100644 --- a/fog/ledger/connection/src/router_client.rs +++ b/fog/ledger/connection/src/router_client.rs @@ -136,12 +136,10 @@ impl LedgerGrpcClient { ) -> Result { log::trace!(self.logger, "Check key images was called"); if !self.is_attested() { - log::warn!(self.logger, "attesting now"); let verification_report = self.attest().await; verification_report?; } - log::warn!(self.logger, "attested"); let key_images_queries = key_images .iter() .map(|&key_image| KeyImageQuery { @@ -149,7 +147,6 @@ impl LedgerGrpcClient { start_block: 0, }) .collect(); - log::warn!(self.logger, "queries are: {:?}", key_images_queries); let key_images_request = CheckKeyImagesRequest { queries: key_images_queries, }; @@ -168,9 +165,7 @@ impl LedgerGrpcClient { let plaintext_bytes = mc_util_serial::encode(&key_images_request); - log::warn!(self.logger, "about to encrypt"); let request_ciphertext = attest_cipher.encrypt(&aad, &plaintext_bytes)?; - log::warn!(self.logger, "encrypted"); msg.set_data(request_ciphertext); msg }; @@ -181,14 +176,12 @@ impl LedgerGrpcClient { .send((request.clone(), grpcio::WriteFlags::default())) .await?; - log::warn!(self.logger, "about to receive"); let message = self .response_receiver .try_next() .await? .ok_or(Error::ResponseNotReceived)? .take_check_key_image_response(); - log::warn!(self.logger, "received"); { let attest_cipher = self @@ -196,9 +189,7 @@ impl LedgerGrpcClient { .as_mut() .expect("no enclave_connection even though attest succeeded"); - log::warn!(self.logger, "about to decrypt"); let plaintext_bytes = attest_cipher.decrypt(message.get_aad(), message.get_data())?; - log::warn!(self.logger, "about to decode"); let plaintext_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes)?; Ok(plaintext_response) From ee1edb2adbf6665be47a42e9bae12f16487d87f3 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Mon, 20 Feb 2023 23:11:09 -0800 Subject: [PATCH 05/11] All Ledger tests now use portpicker to select ports --- fog/ledger/server/tests/connection.rs | 17 ++---- fog/ledger/server/tests/router_connection.rs | 55 +++++++++++--------- fog/ledger/server/tests/store.rs | 6 ++- fog/ledger/server/tests/store_tests.rs | 4 +- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/fog/ledger/server/tests/connection.rs b/fog/ledger/server/tests/connection.rs index 8d2eec72c6..58251308f9 100644 --- a/fog/ledger/server/tests/connection.rs +++ b/fog/ledger/server/tests/connection.rs @@ -32,6 +32,7 @@ use mc_util_from_random::FromRandom; use mc_util_grpc::{GrpcRetryConfig, CHAIN_ID_MISMATCH_ERR_MSG}; use mc_util_test_helper::{CryptoRng, RngCore, RngType, SeedableRng}; use mc_watcher::watcher_db::WatcherDB; +use portpicker::pick_unused_port; use std::{path::PathBuf, str::FromStr, sync::Arc, thread::sleep, time::Duration}; use tempdir::TempDir; use url::Url; @@ -59,8 +60,6 @@ fn setup_watcher_db(logger: Logger) -> (WatcherDB, PathBuf) { // hitting a fog ledger server #[test_with_logger] fn fog_ledger_merkle_proofs_test(logger: Logger) { - let base_port = 3230; - let mut rng = RngType::from_seed([0u8; 32]); for block_version in BlockVersion::iterator() { @@ -111,7 +110,7 @@ fn fog_ledger_merkle_proofs_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports"), )) .unwrap(); let config = LedgerServerConfig { @@ -268,8 +267,6 @@ fn fog_ledger_merkle_proofs_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_key_images_test(logger: Logger) { - let base_port = 3240; - let mut rng = RngType::from_seed([0u8; 32]); for block_version in BlockVersion::iterator() { @@ -342,7 +339,7 @@ fn fog_ledger_key_images_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports") )) .unwrap(); let config = LedgerServerConfig { @@ -484,8 +481,6 @@ fn fog_ledger_key_images_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_blocks_api_test(logger: Logger) { - let base_port = 3250; - let mut rng = RngType::from_seed([0u8; 32]); let alice = AccountKey::random_with_fog(&mut rng); @@ -544,7 +539,7 @@ fn fog_ledger_blocks_api_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports") )) .unwrap(); let config = LedgerServerConfig { @@ -646,8 +641,6 @@ fn fog_ledger_blocks_api_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { - let base_port = 3260; - let mut rng = RngType::from_seed([0u8; 32]); let alice = AccountKey::random_with_fog(&mut rng); @@ -706,7 +699,7 @@ fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { // Make LedgerServer let client_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports") )) .unwrap(); let config = LedgerServerConfig { diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index 853bb8ba68..f1560104fc 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -38,6 +38,7 @@ use mc_util_grpc::{ConnectionUriGrpcioChannel, GrpcRetryConfig, CHAIN_ID_MISMATC use mc_util_test_helper::{CryptoRng, RngCore, RngType, SeedableRng}; use mc_util_uri::AdminUri; use mc_watcher::watcher_db::WatcherDB; +use portpicker::pick_unused_port; use std::{ collections::HashMap, path::PathBuf, @@ -72,8 +73,6 @@ fn setup_watcher_db(logger: Logger) -> (WatcherDB, PathBuf) { // hitting a fog ledger server #[test_with_logger] fn fog_ledger_merkle_proofs_test(logger: Logger) { - let base_port = 3230; - let mut rng = RngType::from_seed([0u8; 32]); for block_version in BlockVersion::iterator() { @@ -124,11 +123,14 @@ fn fog_ledger_merkle_proofs_test(logger: Logger) { // Make LedgerServer let client_listen_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports"), + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") )) .unwrap(); - let admin_listen_uri = - AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); let config = LedgerRouterConfig { chain_id: "local".to_string(), ledger_db: db_full_path.to_path_buf(), @@ -281,8 +283,6 @@ fn fog_ledger_merkle_proofs_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_key_images_test(logger: Logger) { - let base_port = 3240; - let mut rng = RngType::from_seed([0u8; 32]); for block_version in BlockVersion::iterator() { @@ -355,12 +355,14 @@ fn fog_ledger_key_images_test(logger: Logger) { // Make Key Image Store let store_uri = KeyImageStoreUri::from_str(&format!( "insecure-key-image-store://127.0.0.1:{}", - base_port + 9 + pick_unused_port().expect("No free ports") + )) + .unwrap(); + let store_admin_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") )) .unwrap(); - let store_admin_uri = - AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 10)) - .unwrap(); let store_config = LedgerStoreConfig { chain_id: "local".to_string(), client_responder_id: store_uri @@ -410,11 +412,14 @@ fn fog_ledger_key_images_test(logger: Logger) { // Make Router Server let client_listen_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports"), + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") )) .unwrap(); - let admin_listen_uri = - AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); let router_config = LedgerRouterConfig { chain_id: "local".to_string(), ledger_db: db_full_path.to_path_buf(), @@ -546,8 +551,6 @@ fn fog_ledger_key_images_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_blocks_api_test(logger: Logger) { - let base_port = 3250; - let mut rng = RngType::from_seed([0u8; 32]); let alice = AccountKey::random_with_fog(&mut rng); @@ -606,11 +609,14 @@ fn fog_ledger_blocks_api_test(logger: Logger) { // Make LedgerServer let client_listen_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports") + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") )) .unwrap(); - let admin_listen_uri = - AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); let config = LedgerRouterConfig { chain_id: "local".to_string(), ledger_db: db_full_path.to_path_buf(), @@ -708,8 +714,6 @@ fn fog_ledger_blocks_api_test(logger: Logger) { // a fog ledger server #[test_with_logger] fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { - let base_port = 3260; - let mut rng = RngType::from_seed([0u8; 32]); let alice = AccountKey::random_with_fog(&mut rng); @@ -768,11 +772,14 @@ fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { // Make LedgerServer let client_listen_uri = FogLedgerUri::from_str(&format!( "insecure-fog-ledger://127.0.0.1:{}", - base_port + 7 + pick_unused_port().expect("No free ports") + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") )) .unwrap(); - let admin_listen_uri = - AdminUri::from_str(&format!("insecure-mca://127.0.0.1:{}", base_port + 8)).unwrap(); let config = LedgerRouterConfig { chain_id: "local".to_string(), ledger_db: db_full_path.to_path_buf(), diff --git a/fog/ledger/server/tests/store.rs b/fog/ledger/server/tests/store.rs index 883850c6f5..aa1d4f1cb8 100644 --- a/fog/ledger/server/tests/store.rs +++ b/fog/ledger/server/tests/store.rs @@ -35,6 +35,7 @@ use mc_util_uri::UriScheme; use mc_watcher::watcher_db::WatcherDB; use aes_gcm::Aes256Gcm; +use portpicker::pick_unused_port; use sha2::Sha512; use tempdir::TempDir; use url::Url; @@ -150,9 +151,10 @@ lazy_static::lazy_static! { #[test_with_logger] pub fn direct_key_image_store_check(logger: Logger) { const TEST_NAME: &str = "direct_key_image_store_check"; - const PORT_START: u16 = 3223; const OMAP_CAPACITY: u64 = 768; + let port = pick_unused_port().expect("No free ports"); + let rng = RngType::from_entropy(); let TestingContext { enclave, @@ -164,7 +166,7 @@ pub fn direct_key_image_store_check(logger: Logger) { watcher, store_config, watcher_path: _watcher_path, - } = TestingContext::new(TEST_NAME, logger.clone(), PORT_START, OMAP_CAPACITY, rng); + } = TestingContext::new(TEST_NAME, logger.clone(), port, OMAP_CAPACITY, rng); let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index ce8616bd26..92f0b93965 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -37,6 +37,7 @@ use mc_util_uri::UriScheme; use mc_watcher::watcher_db::WatcherDB; use aes_gcm::Aes256Gcm; +use portpicker::pick_unused_port; use sha2::Sha512; use tempdir::TempDir; use url::Url; @@ -152,10 +153,9 @@ lazy_static::lazy_static! { #[test_with_logger] pub fn direct_key_image_store_check(logger: Logger) { const TEST_NAME: &str = "direct_key_image_store_check"; - const PORT_START: u16 = 3223; const OMAP_CAPACITY: u64 = 768; - let port = PORT_START; + let port = pick_unused_port().expect("No free ports"); let rng = RngType::from_entropy(); let TestingContext { From 0e96fb672006c0585f3eeee4490db3de94214f8b Mon Sep 17 00:00:00 2001 From: Emily C Date: Tue, 21 Feb 2023 16:34:50 -0500 Subject: [PATCH 06/11] Fog router support for the unary API (#3123) * Cargo fmt * Ensure unary key image service gets started for router server. * Improving comment clarity. * Apply suggestions removing unnecessary comments Co-authored-by: Nick Santana --------- Co-authored-by: Nick Santana --- fog/ledger/server/src/router_handlers.rs | 4 +- fog/ledger/server/src/router_server.rs | 9 +- fog/ledger/server/src/router_service.rs | 103 ++++++++++++++++++++++- 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/fog/ledger/server/src/router_handlers.rs b/fog/ledger/server/src/router_handlers.rs index 217ddcd412..11adc457aa 100644 --- a/fog/ledger/server/src/router_handlers.rs +++ b/fog/ledger/server/src/router_handlers.rs @@ -156,7 +156,7 @@ pub fn process_shard_responses( } /// Handles a client's authentication request. -fn handle_auth_request( +pub(crate) fn handle_auth_request( enclave: E, auth_message: attest::AuthMessage, logger: Logger, @@ -174,7 +174,7 @@ where } /// Handles a client's query request. -async fn handle_query_request( +pub(crate) async fn handle_query_request( query: attest::Message, enclave: E, shard_clients: Vec>, diff --git a/fog/ledger/server/src/router_server.rs b/fog/ledger/server/src/router_server.rs index 45da1e4c7e..03b01d32d5 100644 --- a/fog/ledger/server/src/router_server.rs +++ b/fog/ledger/server/src/router_server.rs @@ -87,14 +87,18 @@ where // Build our router server. // Init ledger router service. - let ledger_router_service = ledger_grpc::create_ledger_api(LedgerRouterService::new( + let ledger_service = LedgerRouterService::new( enclave.clone(), shards.clone(), config.query_retries, logger.clone(), - )); + ); + + let ledger_router_service = ledger_grpc::create_ledger_api(ledger_service.clone()); log::debug!(logger, "Constructed Ledger Router GRPC Service"); + let unary_key_image_service = ledger_grpc::create_fog_key_image_api(ledger_service); + // Init ledger router admin service. let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( LedgerRouterAdminService::new(shards, logger.clone()), @@ -138,6 +142,7 @@ where let router_server = grpcio::ServerBuilder::new(env.clone()) .register_service(ledger_router_service) + .register_service(unary_key_image_service) .register_service(merkle_proof_service) .register_service(untrusted_tx_out_service) .register_service(block_service) diff --git a/fog/ledger/server/src/router_service.rs b/fog/ledger/server/src/router_service.rs index de61d241ca..a2c50d4c47 100644 --- a/fog/ledger/server/src/router_service.rs +++ b/fog/ledger/server/src/router_service.rs @@ -1,16 +1,20 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use crate::{router_handlers, SVC_COUNTERS}; +use crate::{ + router_handlers::{self, handle_auth_request, handle_query_request}, + SVC_COUNTERS, +}; use futures::{FutureExt, TryFutureExt}; -use grpcio::{DuplexSink, RequestStream, RpcContext}; +use grpcio::{DuplexSink, RequestStream, RpcContext, UnarySink}; +use mc_attest_api::attest::{AuthMessage, Message}; use mc_common::logger::{log, Logger}; use mc_fog_api::{ ledger::{LedgerRequest, LedgerResponse}, - ledger_grpc::{self, LedgerApi}, + ledger_grpc::{self, FogKeyImageApi, KeyImageStoreApiClient, LedgerApi}, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::KeyImageStoreUri; -use mc_util_grpc::rpc_logger; +use mc_util_grpc::{rpc_internal_error, rpc_logger}; use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -80,3 +84,94 @@ where }); } } + +/// Used for the implementation of FogKeyImageApi::check_key_images(), +/// the legacy unary key-image API, for LedgerRouterService. +async fn unary_check_key_image_inner( + request: Message, + query_retries: usize, + enclave: E, + sink: UnarySink, + shard_clients: Vec>, + scope_logger: Logger, +) -> Result<(), grpcio::Error> +where + E: LedgerEnclaveProxy, +{ + let result = handle_query_request( + request, + enclave, + shard_clients, + query_retries, + scope_logger.clone(), + ) + .await; + + match result { + Ok(mut response) => { + if response.has_check_key_image_response() { + sink.success(response.take_check_key_image_response()).await + } else { + let error = rpc_internal_error( + "Inavlid LedgerRequest response", + "No check key image response to client's key image request.".to_string(), + &scope_logger, + ); + sink.fail(error).await + } + } + Err(rpc_status) => sink.fail(rpc_status).await, + } +} + +// This API is the unary key-image-specific equivalent of LedgerApi. +impl FogKeyImageApi for LedgerRouterService { + fn check_key_images(&mut self, ctx: RpcContext, request: Message, sink: UnarySink) { + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + let logger = logger.clone(); + let shards = self.shards.read().expect("RwLock poisoned"); + + let future = unary_check_key_image_inner( + request, + self.query_retries, + self.enclave.clone(), + sink, + shards.values().cloned().collect(), + logger.clone(), + ) + .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) + // TODO: Do more with the error than just push it to the log. + .map(|_| ()); + + ctx.spawn(future); + }) + } + + fn auth(&mut self, ctx: RpcContext, request: AuthMessage, sink: UnarySink) { + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + let logger = logger.clone(); + let result = handle_auth_request(self.enclave.clone(), request, logger.clone()); + let future = match result { + Ok(mut response) => { + if response.has_auth() { + sink.success(response.take_auth()) + } else { + let error = rpc_internal_error( + "Inavlid LedgerRequest response", + "Response to client's auth request did not contain an auth response." + .to_string(), + &logger, + ); + sink.fail(error) + } + } + Err(rpc_status) => sink.fail(rpc_status), + } + .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) + .map(|_| ()); + ctx.spawn(future); + }); + } +} From 109f477f16866ba71996224d41c43231d433f54d Mon Sep 17 00:00:00 2001 From: Millie C Date: Thu, 23 Feb 2023 17:58:38 -0500 Subject: [PATCH 07/11] Test key image retrieval via unary API on the router. --- fog/ledger/server/tests/router_connection.rs | 277 ++++++++++++++++++- 1 file changed, 276 insertions(+), 1 deletion(-) diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index f1560104fc..15afa2f100 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -19,7 +19,7 @@ use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; use mc_fog_api::{ledger::TxOutResultCode, ledger_grpc::KeyImageStoreApiClient}; use mc_fog_ledger_connection::{ Error, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, KeyImageResultExtension, - LedgerGrpcClient, OutputResultExtension, + LedgerGrpcClient, OutputResultExtension, FogKeyImageGrpcClient, }; use mc_fog_ledger_enclave::LedgerSgxEnclave; use mc_fog_ledger_server::{ @@ -863,6 +863,281 @@ fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { sleep(Duration::from_millis(1000)); } + +// Test that a fog ledger connection is able to check key images by hitting +// a fog ledger router using the unary API +#[test_with_logger] +fn fog_router_unary_key_image_test(logger: Logger) { + let mut rng = RngType::from_seed([0u8; 32]); + + for block_version in BlockVersion::iterator() { + let alice = AccountKey::random_with_fog(&mut rng); + + let recipients = vec![alice.default_subaddress()]; + + let keys: Vec = (0..20).map(|x| KeyImage::from(x as u64)).collect(); + + // Make LedgerDB + let ledger_dir = TempDir::new("fog-ledger").expect("Could not get test_ledger tempdir"); + let db_full_path = ledger_dir.path(); + let mut ledger = recreate_ledger_db(db_full_path); + + // Make WatcherDB + let (mut watcher, watcher_dir) = setup_watcher_db(logger.clone()); + + // Populate ledger with some data + // Origin block cannot have key images + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &[], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[0..2], + &mut rng, + &mut watcher, + ); + add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[3..6], + &mut rng, + &mut watcher, + ); + let num_blocks = add_block_to_ledger( + block_version, + &mut ledger, + &recipients, + &keys[6..9], + &mut rng, + &mut watcher, + ); + + // Populate watcher with Signature and Timestamp for block 1 + let url1 = Url::parse(TEST_URL).unwrap(); + let block1 = ledger.get_block(1).unwrap(); + let signing_key_a = Ed25519Pair::from_random(&mut rng); + let filename = String::from("00/00"); + let mut signed_block_a1 = + BlockSignature::from_block_and_keypair(&block1, &signing_key_a).unwrap(); + signed_block_a1.set_signed_at(1593798844); + watcher + .add_block_signature(&url1, 1, signed_block_a1, filename.clone()) + .unwrap(); + + // Update last synced to block 2, to indicate that this URL did not participate + // in consensus for block 2. + watcher.update_last_synced(&url1, 2).unwrap(); + + { + // Make Key Image Store + let store_uri = KeyImageStoreUri::from_str(&format!( + "insecure-key-image-store://127.0.0.1:{}", + pick_unused_port().expect("No free ports") + )) + .unwrap(); + let store_admin_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") + )) + .unwrap(); + let store_config = LedgerStoreConfig { + chain_id: "local".to_string(), + client_responder_id: store_uri + .responder_id() + .expect("Couldn't get responder ID for store"), + client_listen_uri: store_uri.clone(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir.clone(), + ias_api_key: Default::default(), + ias_spid: Default::default(), + admin_listen_uri: Some(store_admin_uri), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + omap_capacity: OMAP_CAPACITY, + sharding_strategy: ShardingStrategy::Epoch(EpochShardingStrategy::default()), + }; + let store_enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &store_config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + let ra_client = + AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); + let mut store_server = KeyImageStoreServer::new_from_config( + store_config, + store_enclave, + ra_client, + ledger.clone(), + watcher.clone(), + EpochShardingStrategy::default(), + SystemTimeProvider::default(), + logger.clone(), + ); + + // Make Key Image Store client + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let store_client = KeyImageStoreApiClient::new( + ChannelBuilder::default_channel_builder(grpc_env.clone()) + .connect_to_uri(&store_uri, &logger), + ); + let mut store_clients = HashMap::new(); + store_clients.insert(store_uri, Arc::new(store_client)); + let shards = Arc::new(RwLock::new(store_clients)); + + // Make Router Server + let router_client_listen_uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://127.0.0.1:{}", + pick_unused_port().expect("No free ports"), + )) + .unwrap(); + let admin_listen_uri = AdminUri::from_str(&format!( + "insecure-mca://127.0.0.1:{}", + pick_unused_port().expect("No free ports") + )) + .unwrap(); + let router_config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: db_full_path.to_path_buf(), + watcher_db: watcher_dir, + admin_listen_uri: admin_listen_uri.clone(), + client_listen_uri: router_client_listen_uri.clone(), + client_responder_id: ResponderId::from_str(&router_client_listen_uri.addr()).unwrap(), + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: OMAP_CAPACITY, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &router_config.client_responder_id, + OMAP_CAPACITY, + logger.clone(), + ); + + let ra_client = + AttestClient::new(&router_config.ias_api_key).expect("Could not create IAS client"); + let mut router_server = LedgerRouterServer::new( + router_config, + enclave, + ra_client, + shards, + ledger.clone(), + watcher.clone(), + logger.clone(), + ); + + store_server.start(); + router_server.start(); + + // Make ledger enclave client + let mut mr_signer_verifier = + MrSignerVerifier::from(mc_fog_ledger_enclave_measurement::sigstruct()); + mr_signer_verifier.allow_hardening_advisories( + mc_fog_ledger_enclave_measurement::HARDENING_ADVISORIES, + ); + + let mut verifier = Verifier::default(); + verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); + + let mut client = FogKeyImageGrpcClient::new( + String::default(), + router_client_listen_uri, + GRPC_RETRY_CONFIG, + verifier, + grpc_env, + logger.clone(), + ); + + // Check on key images + let mut response = client + .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) + .expect("check_key_images failed"); + + let mut n = 1; + + while response.num_blocks != num_blocks { + response = client + .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) + .expect("check_key_images failed"); + + // Ideally this should not require a sleep, but that's for a later PR. + sleep(Duration::from_secs(10)); + // panic on the 20th time + n += 1; // + if n > 20 { + panic!("Fog ledger not fully initialized"); + } + } + + // FIXME assert_eq!(response.num_txos, ...); + assert_eq!(response.results[0].key_image, keys[0]); + assert_eq!(response.results[0].status(), Ok(Some(1))); + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[0].timestamp, 1); + + assert_eq!(response.results[1].key_image, keys[1]); + assert_eq!(response.results[1].status(), Ok(Some(1))); + assert_eq!( + response.results[1].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[1].timestamp, 1); + + // Check a key_image for a block which will never have signatures & timestamps + assert_eq!(response.results[2].key_image, keys[3]); + assert_eq!(response.results[2].status(), Ok(Some(2))); // Spent in block 2 + assert_eq!( + response.results[2].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[2].timestamp, 2); + + // Watcher has only synced 1 block, so timestamp should be behind + assert_eq!(response.results[3].key_image, keys[7]); + assert_eq!(response.results[3].status(), Ok(Some(3))); // Spent in block 3 + assert_eq!( + response.results[3].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[3].timestamp, 3); + + // Check a key_image that has not been spent + assert_eq!(response.results[4].key_image, keys[19]); + assert_eq!(response.results[4].status(), Ok(None)); // Not spent + assert_eq!( + response.results[4].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + assert_eq!(response.results[4].timestamp, u64::MAX); + } + + // FIXME: Check a key_image that generates a DatabaseError - tough to generate + + // grpcio detaches all its threads and does not join them :( + // we opened a PR here: https://github.com/tikv/grpc-rs/pull/455 + // in the meantime we can just sleep after grpcio env and all related + // objects have been destroyed, and hope that those 6 threads see the + // shutdown requests within 1 second. + sleep(Duration::from_millis(1000)); + } +} + // Infra /// Adds a block containing one txo for each provided recipient and returns new From 3d331cf794e255270d5232f4ac8a0574d2aeb5df Mon Sep 17 00:00:00 2001 From: Millie C Date: Fri, 24 Feb 2023 13:16:41 -0500 Subject: [PATCH 08/11] Cargo fmt --- fog/ledger/server/tests/router_connection.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index 15afa2f100..2f763916ad 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -18,8 +18,8 @@ use mc_common::{ use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; use mc_fog_api::{ledger::TxOutResultCode, ledger_grpc::KeyImageStoreApiClient}; use mc_fog_ledger_connection::{ - Error, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, KeyImageResultExtension, - LedgerGrpcClient, OutputResultExtension, FogKeyImageGrpcClient, + Error, FogKeyImageGrpcClient, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, + KeyImageResultExtension, LedgerGrpcClient, OutputResultExtension, }; use mc_fog_ledger_enclave::LedgerSgxEnclave; use mc_fog_ledger_server::{ @@ -863,7 +863,6 @@ fn fog_ledger_untrusted_tx_out_api_test(logger: Logger) { sleep(Duration::from_millis(1000)); } - // Test that a fog ledger connection is able to check key images by hitting // a fog ledger router using the unary API #[test_with_logger] @@ -1011,7 +1010,8 @@ fn fog_router_unary_key_image_test(logger: Logger) { watcher_db: watcher_dir, admin_listen_uri: admin_listen_uri.clone(), client_listen_uri: router_client_listen_uri.clone(), - client_responder_id: ResponderId::from_str(&router_client_listen_uri.addr()).unwrap(), + client_responder_id: ResponderId::from_str(&router_client_listen_uri.addr()) + .unwrap(), ias_spid: Default::default(), ias_api_key: Default::default(), client_auth_token_secret: None, @@ -1073,7 +1073,7 @@ fn fog_router_unary_key_image_test(logger: Logger) { .check_key_images(&[keys[0], keys[1], keys[3], keys[7], keys[19]]) .expect("check_key_images failed"); - // Ideally this should not require a sleep, but that's for a later PR. + // Ideally this should not require a sleep, but that's for a later PR. sleep(Duration::from_secs(10)); // panic on the 20th time n += 1; // From d87be15b7c64d313744a76ecd61aa1ae52efdad1 Mon Sep 17 00:00:00 2001 From: Millie C Date: Mon, 27 Feb 2023 22:08:34 -0500 Subject: [PATCH 09/11] Fix a merging mistake. --- fog/ledger/server/tests/router_connection.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index d78da76350..bb9099d10d 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -1019,7 +1019,6 @@ fn fog_router_unary_key_image_test(logger: Logger) { client_responder_id: router_client_listen_uri .responder_id() .expect("Couldn't get responder ID for router"), - .unwrap(), ias_spid: Default::default(), ias_api_key: Default::default(), client_auth_token_secret: None, From 239cf328c3a44607cfc58c2f1103f206ac0ec29f Mon Sep 17 00:00:00 2001 From: Millie C Date: Mon, 27 Feb 2023 23:02:40 -0500 Subject: [PATCH 10/11] Fix some additional merge mistakes --- fog/ledger/server/tests/connection.rs | 2 +- fog/ledger/server/tests/router_connection.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fog/ledger/server/tests/connection.rs b/fog/ledger/server/tests/connection.rs index 6ae42e4319..5c7b43f9d1 100644 --- a/fog/ledger/server/tests/connection.rs +++ b/fog/ledger/server/tests/connection.rs @@ -32,7 +32,7 @@ use mc_util_from_random::FromRandom; use mc_util_grpc::{GrpcRetryConfig, CHAIN_ID_MISMATCH_ERR_MSG}; use mc_util_test_helper::{CryptoRng, RngCore, RngType, SeedableRng}; use mc_watcher::watcher_db::WatcherDB; -use portpicker::pick_unused_port; + use std::{path::PathBuf, str::FromStr, sync::Arc, thread::sleep, time::Duration}; use tempdir::TempDir; use url::Url; diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index bb9099d10d..1f0ee1fb7a 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -18,7 +18,7 @@ use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; use mc_fog_api::{ledger::TxOutResultCode, ledger_grpc::KeyImageStoreApiClient}; use mc_fog_ledger_connection::{ Error, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, KeyImageResultExtension, - LedgerGrpcClient, OutputResultExtension, + LedgerGrpcClient, OutputResultExtension, FogKeyImageGrpcClient, }; use mc_fog_ledger_enclave::LedgerSgxEnclave; use mc_fog_ledger_server::{ From f62a0176569cb9fa3cfaa1c821e2b89416c49c4e Mon Sep 17 00:00:00 2001 From: Millie C Date: Tue, 28 Feb 2023 14:36:19 -0500 Subject: [PATCH 11/11] Sort dependencies --- fog/ledger/server/tests/router_connection.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fog/ledger/server/tests/router_connection.rs b/fog/ledger/server/tests/router_connection.rs index 1f0ee1fb7a..96295cf4eb 100644 --- a/fog/ledger/server/tests/router_connection.rs +++ b/fog/ledger/server/tests/router_connection.rs @@ -17,8 +17,8 @@ use mc_common::{ use mc_crypto_keys::{CompressedRistrettoPublic, Ed25519Pair}; use mc_fog_api::{ledger::TxOutResultCode, ledger_grpc::KeyImageStoreApiClient}; use mc_fog_ledger_connection::{ - Error, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, KeyImageResultExtension, - LedgerGrpcClient, OutputResultExtension, FogKeyImageGrpcClient, + Error, FogKeyImageGrpcClient, FogMerkleProofGrpcClient, FogUntrustedLedgerGrpcClient, + KeyImageResultExtension, LedgerGrpcClient, OutputResultExtension, }; use mc_fog_ledger_enclave::LedgerSgxEnclave; use mc_fog_ledger_server::{