From 7e2b34b3da71a686314e0603c43622280329aae7 Mon Sep 17 00:00:00 2001 From: awygle Date: Tue, 27 Sep 2022 13:15:56 -0700 Subject: [PATCH] Ledger router server prototype (#2612) * Create helper methods to obliviously collate shard responses (#2252) * Create helper methods that obliviously collate shard responses * Implement chris's suggestions * Fix error string * Fix lint * Sort dependencies * Run format * fix newline * Collate shard response in constant time (obliviously) (#2250) * Add backend decrypt method to ake enclave * Add enclave methods for shard response collation * Refactor response to always include view store uri * Use shard responses to fill router response * Implement Nick's suggestions and fix CI errors * Implement epoch sharding via trait (#2350) * Use sharding strategy to tell view what blocks to process (#2351) * Return minimum highest processed block to client (#2387) * Add config option for sharding strategy (#2352) * Add readiness API to ShardingStrategy (#2353) * Add readiness API to ShardingStrategy * Implement readiness check per ShardingStrategy * Change MVQR error to accommodate different types * Set not ready message * Pull out query processing logic to separate method * Add report fetching logic to FVR (#2454) * Fix responder ID usage in FVR (#2446) * Create different types of View Servers for store and client-facing (#2447) * Messages for noise protocol exchanges with explicit nonces (#2461) * attest::NonceMessage, EnclaveNonceMessage, add handling to attest-ake * Rustfmt fixes * Minor DRY cleanups around sessions and auth messages (#2462) * attest::NonceMessage, EnclaveNonceMessage, add handling to attest-ake * Rustfmt fixes * Minor DRY cleanups around sessions and auth messages * Update FVR client for integration test (#2448) * Create gRPC FogViewRouterAdminAPI (#2360) * Create FogViewRouterAdminUri (#2361) * Add NonceAuthRequest, NonceAuthResponse, NonceSession (#2463) * Add NonceAuthRequest, NonceAuthResponse, NonceSession * Remove EnclaveNonceMessage in favor of EnclaveMessage * Manually implement Hash to resolve clippy warning * Implement client message to sealed message API for backends * Update Fog View enclave to build with new AKE enclave API * Rust type aliases don't protect against passing the aliased type, use a new type instead * Re-add comment ASCII art * Additional plumbing of new SealedClientMessage type * Move SealedClientMessage, finish implementing in fog view router * Properly update the lockfiles * Fix clippy lint * Starting branch for ledger router service. * Adapted more of fog view router code for ledger router. * Revert mistake in shard_responses_processor.rs * Adapting copied fog view router code, continuing to fill in a first draft. * Ledger router bin improvements. * EnclaveCall changes * Adding ledger_store_server and service * Adding new enclave methods to ecall_dispatcher() * Clarifying names * Further disentangling of parts of the ledger, plus filling out more enclave-related code * More reorganization * Rewrite create_key_image_store_query to use sealed messages (without changing logic) * Plumb decrypt_and_seal_query enclave call * Enable enclave call for multi-query creation * Implement shard response processing as in Fog View Router * Enable store authentication from router * Implement retry logic * First-pass implementation of response collation Co-authored-by: Sam Dealy <33067698+samdealy@users.noreply.github.com> Co-authored-by: James Cape Co-authored-by: Millie C Co-authored-by: NotGyro --- Cargo.lock | 11 ++ fog/api/proto/ledger.proto | 24 ++- fog/api/src/conversions.rs | 13 +- fog/ledger/connection/src/error.rs | 7 + fog/ledger/enclave/api/src/lib.rs | 47 +++-- fog/ledger/enclave/api/src/messages.rs | 46 +++-- fog/ledger/enclave/impl/Cargo.toml | 1 + fog/ledger/enclave/impl/src/lib.rs | 117 +++++++++-- fog/ledger/enclave/src/lib.rs | 64 +++++- fog/ledger/enclave/trusted/Cargo.lock | 36 ++++ fog/ledger/enclave/trusted/src/lib.rs | 29 ++- fog/ledger/server/Cargo.toml | 2 +- fog/ledger/server/src/bin/key_image_router.rs | 18 +- fog/ledger/server/src/config.rs | 10 +- fog/ledger/server/src/error.rs | 12 +- .../server/src/key_image_router_server.rs | 21 +- .../server/src/key_image_router_service.rs | 24 ++- fog/ledger/server/src/key_image_service.rs | 20 +- fog/ledger/server/src/lib.rs | 7 +- fog/ledger/server/src/router_handlers.rs | 186 +++++++++++------- fog/ledger/test_infra/src/lib.rs | 7 +- fog/view/connection/src/bin/router_client.rs | 26 --- fog/view/enclave/trusted/Cargo.lock | 1 + fog/view/server/src/config.rs | 8 + 24 files changed, 536 insertions(+), 201 deletions(-) delete mode 100644 fog/view/connection/src/bin/router_client.rs diff --git a/Cargo.lock b/Cargo.lock index 92b24a0b2b..2aee910677 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3701,6 +3701,7 @@ dependencies = [ "aligned-cmov", "mc-attest-core", "mc-attest-enclave-api", + "mc-blockchain-types", "mc-common", "mc-crypto-ake-enclave", "mc-crypto-keys", @@ -4296,13 +4297,17 @@ dependencies = [ name = "mc-fog-view-connection" version = "1.3.0-pre0" dependencies = [ + "aes-gcm", "futures", "grpcio", + "mc-attest-ake", "mc-attest-api", "mc-attest-core", "mc-attest-verifier", "mc-common", "mc-crypto-keys", + "mc-crypto-noise", + "mc-crypto-rand", "mc-fog-api", "mc-fog-enclave-connection", "mc-fog-types", @@ -4311,7 +4316,9 @@ dependencies = [ "mc-util-grpc", "mc-util-serial", "mc-util-telemetry", + "mc-util-uri", "retry", + "sha2 0.10.2", "tokio", ] @@ -4325,6 +4332,7 @@ dependencies = [ "mc-attest-enclave-api", "mc-attest-verifier", "mc-common", + "mc-crypto-ake-enclave", "mc-crypto-keys", "mc-enclave-boundary", "mc-fog-ocall-oram-storage-edl", @@ -4358,6 +4366,7 @@ dependencies = [ "mc-attest-core", "mc-attest-enclave-api", "mc-common", + "mc-crypto-ake-enclave", "mc-crypto-keys", "mc-crypto-noise", "mc-fog-recovery-db-iface", @@ -4383,6 +4392,7 @@ version = "1.3.0-pre0" dependencies = [ "aes-gcm", "aligned-cmov", + "itertools", "mc-attest-ake", "mc-attest-core", "mc-attest-enclave-api", @@ -4468,6 +4478,7 @@ dependencies = [ "lazy_static", "mc-attest-api", "mc-attest-core", + "mc-attest-enclave-api", "mc-attest-net", "mc-attest-verifier", "mc-blockchain-types", diff --git a/fog/api/proto/ledger.proto b/fog/api/proto/ledger.proto index 908be70ade..ff0edbf6db 100644 --- a/fog/api/proto/ledger.proto +++ b/fog/api/proto/ledger.proto @@ -60,15 +60,33 @@ message MultiKeyImageStoreRequest { repeated attest.Message queries = 1; } + +/// The status associated with a MultiViewStoreQueryResponse +enum MultiKeyImageStoreResponseStatus { + /// The Fog Ledger Store successfully fulfilled the request. + SUCCESS = 0; + /// The Fog Ledger Store is unable to decrypt a query within the MultiKeyImageStoreRequest. It needs to be authenticated + /// by the router. + AUTHENTICATION_ERROR = 1; + /// The Fog Ledger Store is not ready to service a MultiViewStoreQueryRequest. This might be because the store has + /// not loaded enough blocks yet. + NOT_READY = 2; +} + message MultiKeyImageStoreResponse { /// Optional field that gets set when the Fog Ledger Store is able to decrypt a query /// included in the MultiKeyImageStoreRequest and create a query response for that // query. attest.Message query_response = 1; - /// Optional error that gets returned when the Fog Ledger Store - /// cannot decrypt the MultiKeyImageStoreRequest. - FogLedgerStoreDecryptionError decryption_error = 2; + /// The FogViewStoreUri for the specific Fog View Store that + /// tried to decrypt the MultiViewStoreQueryRequest and failed. + /// The client should subsequently authenticate with the machine + /// described by this URI. + string fog_ledger_store_uri = 2; + + /// Status that gets returned when the Fog Ledger Store services a MultiKeyImageStoreRequest. + MultiKeyImageStoreResponseStatus status = 3; } //// diff --git a/fog/api/src/conversions.rs b/fog/api/src/conversions.rs index da60b0109a..7550bafcd7 100644 --- a/fog/api/src/conversions.rs +++ b/fog/api/src/conversions.rs @@ -2,7 +2,9 @@ // // Contains helper methods that enable conversions for Fog Api types. -use crate::{fog_common, ingest_common, view::MultiViewStoreQueryRequest}; +use crate::{ + fog_common, ingest_common, ledger::MultiKeyImageStoreRequest, view::MultiViewStoreQueryRequest, +}; use mc_api::ConversionError; use mc_attest_api::attest; use mc_attest_enclave_api::{EnclaveMessage, NonceSession}; @@ -31,6 +33,15 @@ impl From> for MultiViewStoreQueryRequest { } } +impl From> for MultiKeyImageStoreRequest { + fn from(attested_query_messages: Vec) -> MultiKeyImageStoreRequest { + let mut multi_key_image_store_request = MultiKeyImageStoreRequest::new(); + multi_key_image_store_request.set_queries(attested_query_messages.into()); + + multi_key_image_store_request + } +} + impl From<&common::BlockRange> for fog_common::BlockRange { fn from(common_block_range: &common::BlockRange) -> fog_common::BlockRange { let mut proto_block_range = fog_common::BlockRange::new(); diff --git a/fog/ledger/connection/src/error.rs b/fog/ledger/connection/src/error.rs index 3b2e11d2df..2a0cb9238a 100644 --- a/fog/ledger/connection/src/error.rs +++ b/fog/ledger/connection/src/error.rs @@ -8,6 +8,7 @@ use mc_api::ConversionError; use mc_fog_enclave_connection::Error as EnclaveConnectionError; use mc_fog_uri::FogLedgerUri; +use mc_util_uri::UriConversionError; /// Error type returned by LedgerServerConn #[derive(Debug, Display)] @@ -35,3 +36,9 @@ impl From for Error { Error::Conversion(err) } } + +impl From for Error { + fn from(err: UriConversionError) -> Self { + Self::UriConversionError(err) + } +} diff --git a/fog/ledger/enclave/api/src/lib.rs b/fog/ledger/enclave/api/src/lib.rs index edf2115134..c35fb48d41 100644 --- a/fog/ledger/enclave/api/src/lib.rs +++ b/fog/ledger/enclave/api/src/lib.rs @@ -13,9 +13,11 @@ pub use crate::{ error::{AddRecordsError, Error}, messages::{EnclaveCall, KeyImageData}, }; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; use core::result::Result as StdResult; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; use mc_common::ResponderId; use mc_crypto_keys::X25519Public; pub use mc_fog_types::ledger::{ @@ -99,12 +101,13 @@ pub trait LedgerEnclave: ReportableEnclave { /// Add a key image data to the oram Using thrm -rf targete key image fn add_key_image_data(&self, records: Vec) -> Result<()>; - // LEDGER ROUTER / STORE SYSTEM - /// Begin a connection to a Fog Ledger Store. The enclave calling this method, - /// most likely a router, will act as a client to the Fog Ledger Store. - fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) -> Result; + /// Begin a connection to a Fog Ledger Store. The enclave calling this + /// method, most likely a router, will act as a client to the Fog Ledger + /// Store. + fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) + -> Result; /// Complete the connection to a Fog Ledger Store that has accepted our /// ClientAuthRequest. This is meant to be called after the enclave has @@ -114,22 +117,38 @@ pub trait LedgerEnclave: ReportableEnclave { ledger_store_id: ResponderId, ledger_store_auth_response: ClientAuthResponse, ) -> Result<()>; - + + /// Decrypts a client query message and converts it into a + /// SealedClientMessage which can be unsealed multiple times to + /// construct the MultiKeyImageStoreRequest. + fn decrypt_and_seal_query( + &self, + client_query: EnclaveMessage, + ) -> Result; + /// Transforms a client query request into a list of query request data. /// /// The returned list is meant to be used to construct the - /// MultiLedgerStoreQuery, which is sent to each shard. - fn create_key_image_store_query( + /// MultiKeyImageStoreRequest, which is sent to each shard. + fn create_multi_key_image_store_query_data( &self, - client_query: EnclaveMessage, + sealed_query: SealedClientMessage, ) -> Result>>; - /// Used by a Ledger Store to handle an inbound encrypted ledger.proto LedgerRequest. - /// Generally, these come in from a router. - /// This could could be a key image request, a merkele proof + /// Receives all of the shards' query responses and collates them into one + /// query response for the client. + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result>; + + /// Used by a Ledger Store to handle an inbound encrypted ledger.proto + /// LedgerRequest. Generally, these come in from a router. + /// This could could be a key image request, a merkele proof /// request, and potentially in the future an untrusted tx out request. fn handle_key_image_store_request( - &self, + &self, router_query: EnclaveMessage, ) -> Result>; } diff --git a/fog/ledger/enclave/api/src/messages.rs b/fog/ledger/enclave/api/src/messages.rs index 332727a7c5..f4fc4cc0ac 100644 --- a/fog/ledger/enclave/api/src/messages.rs +++ b/fog/ledger/enclave/api/src/messages.rs @@ -2,9 +2,13 @@ //! The message types used by the ledger_enclave_api. use crate::UntrustedKeyImageQueryResponse; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; use mc_attest_core::{Quote, Report, TargetInfo, VerificationReport}; -use mc_attest_enclave_api::{ClientAuthRequest, ClientSession, EnclaveMessage, ClientAuthResponse}; + +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; + use mc_common::ResponderId; use mc_fog_types::ledger::GetOutputsResponse; use mc_transaction_core::ring_signature::KeyImage; @@ -34,7 +38,7 @@ pub struct KeyImageData { /// An enumeration of API calls and their arguments for use across serialization /// boundaries. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum EnclaveCall { /// The [LedgerEnclave::enclave_init()] method. EnclaveInit(ResponderId, u64), @@ -104,8 +108,10 @@ pub enum EnclaveCall { /// The [LedgerEnclave::connect_to_store()] method. /// - /// Begin a connection to a Fog Ledger Store. The enclave calling this method, - /// most likely a router, will act as a client to the Fog Ledger Store. + /// Begin a connection to a Fog Ledger Store. The enclave calling this + /// method, most likely a router, will act as a client to the Fog Ledger + /// Store. + ConnectToKeyImageStore(ResponderId), /// The [LedgerEnclave::finish_connecting_to_store()] method. @@ -113,24 +119,34 @@ pub enum EnclaveCall { /// Complete the connection to a Fog Ledger Store that has accepted our /// ClientAuthRequest. This is meant to be called after the enclave has /// initialized and discovers a new Fog Ledger Store. - FinishConnectingToKeyImageStore( - ResponderId, - ClientAuthResponse, - ), - - /// The [LedgerEnclave::create_key_image_store_query()] method. + FinishConnectingToKeyImageStore(ResponderId, ClientAuthResponse), + + /// The [LedgerEnclave::decrypt_and_seal_query()] method. + /// + /// Takes a client query message and returns a SealedClientMessage + /// sealed for the current enclave. + DecryptAndSealQuery(EnclaveMessage), + + /// The [LedgerEnclave::create_multi_key_image_store_query()] method. /// /// Transforms a client query request into a list of query request data. /// /// The returned list is meant to be used to construct the /// MultiKeyImageStoreRequest, which is sent to each shard. - CreateKeyImageStoreQuery(EnclaveMessage), + CreateMultiKeyImageStoreQueryData(SealedClientMessage), + + /// Collates shard query responses into a single query response for the + /// client. + CollateQueryResponses( + SealedClientMessage, + BTreeMap>, + ), /// The [LedgerEnclave::handle_key_image_store_request()] method. /// - /// Used by a Ledger Store to handle an inbound encrypted ledger.proto LedgerRequest. - /// Generally, these come in from a router. - /// This could could be a key image request, a merkele proof + /// Used by a Ledger Store to handle an inbound encrypted ledger.proto + /// LedgerRequest. Generally, these come in from a router. + /// This could could be a key image request, a merkele proof /// request, and potentially in the future an untrusted tx out request. HandleKeyImageStoreRequest(EnclaveMessage), } \ No newline at end of file diff --git a/fog/ledger/enclave/impl/Cargo.toml b/fog/ledger/enclave/impl/Cargo.toml index 7e4695a471..f93670323d 100644 --- a/fog/ledger/enclave/impl/Cargo.toml +++ b/fog/ledger/enclave/impl/Cargo.toml @@ -12,6 +12,7 @@ license = "GPL-3.0" # mobilecoin mc-attest-core = { path = "../../../../attest/core", default-features = false } mc-attest-enclave-api = { path = "../../../../attest/enclave-api", default-features = false } +mc-blockchain-types = { path = "../../../../blockchain/types" } mc-common = { path = "../../../../common", default-features = false } mc-crypto-ake-enclave = { path = "../../../../crypto/ake/enclave", default-features = false } mc-crypto-keys = { path = "../../../../crypto/keys", default-features = false } diff --git a/fog/ledger/enclave/impl/src/lib.rs b/fog/ledger/enclave/impl/src/lib.rs index 36b73a0a19..325a4d472f 100644 --- a/fog/ledger/enclave/impl/src/lib.rs +++ b/fog/ledger/enclave/impl/src/lib.rs @@ -13,10 +13,14 @@ extern crate alloc; mod key_image_store; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; +use core::cmp::max; use key_image_store::{KeyImageStore, StorageDataSize, StorageMetaSize}; use mc_attest_core::{IasNonce, Quote, QuoteNonce, Report, TargetInfo, VerificationReport}; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; +use mc_blockchain_types::MAX_BLOCK_VERSION; use mc_common::{ logger::{log, Logger}, ResponderId, @@ -192,8 +196,14 @@ where Ok(()) } - fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) -> Result { - mc_sgx_debug::eprintln!("Called connect_to_key_image_store(ledger_store_id: {})", ledger_store_id); + fn connect_to_key_image_store( + &self, + ledger_store_id: ResponderId, + ) -> Result { + mc_sgx_debug::eprintln!( + "Called connect_to_key_image_store(ledger_store_id: {})", + ledger_store_id + ); Ok(self.ake.backend_init(ledger_store_id)?) } @@ -209,20 +219,105 @@ where .backend_connect(ledger_store_id, ledger_store_auth_response)?) } - fn create_key_image_store_query( + fn decrypt_and_seal_query( &self, client_query: EnclaveMessage, + ) -> Result { + Ok(self.ake.decrypt_client_message_for_enclave(client_query)?) + } + + fn create_multi_key_image_store_query_data( + &self, + sealed_query: SealedClientMessage, ) -> Result>> { - mc_sgx_debug::eprintln!("Called create_key_image_store_query(..)"); + mc_sgx_debug::eprintln!("Called create_multi_key_image_store_query_data(..)"); Ok(self .ake - .reencrypt_client_message_for_backends(client_query)?) + .reencrypt_sealed_message_for_backends(&sealed_query)?) } + + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result> { + if shard_query_responses.is_empty() { + return Ok(EnclaveMessage::default()); + } + let channel_id = sealed_query.channel_id.clone(); + let client_query_plaintext = self.ake.unseal(&sealed_query)?; + // TODO this will (possibly?) be used when we implement obliviousness + let _client_query_request: CheckKeyImagesRequest = + mc_util_serial::decode(&client_query_plaintext).map_err(|e| { + log::error!(self.logger, "Could not decode client query request: {}", e); + Error::ProstDecode + })?; + + let shard_query_responses = shard_query_responses + .into_iter() + .map(|(responder_id, enclave_message)| { + let plaintext_bytes = self.ake.backend_decrypt(responder_id, enclave_message)?; // TODO explicit nonces + let query_response: CheckKeyImagesResponse = + mc_util_serial::decode(&plaintext_bytes)?; + + Ok(query_response) + }) + .collect::>>()?; + + // NOTES: + // num_blocks = min(responses.num_blocks) + // global_txo_count = min(global_txo_count) TODO CONFIRM + // results = cat(responses.results) + // latest_block_version = max(responses.latest_block_version) + // max_block_version = max(latest_block_version, + // mc_transaction_core::MAX_BLOCK_VERSION + + // TODO no unwraps + let num_blocks = shard_query_responses + .iter() + .map(|query_response| query_response.num_blocks) + .min() + .unwrap(); + let global_txo_count = shard_query_responses + .iter() + .map(|query_response| query_response.global_txo_count) + .min() + .unwrap(); + let latest_block_version = shard_query_responses + .iter() + .map(|query_response| query_response.latest_block_version) + .max() + .unwrap(); + // TODO I believe this needs to be implemented in an oblivious way to meet the + // security requirements. I'm not 100% sure what an oblivious approach + // to this looks like, though. In general this kind of thing needs to be + // talked about. + let results = shard_query_responses + .into_iter() + .flat_map(|query_response| query_response.results) + .collect(); + let max_block_version = max(latest_block_version, *MAX_BLOCK_VERSION); + + let client_query_response = CheckKeyImagesResponse { + num_blocks, + global_txo_count, + results, + latest_block_version, + max_block_version, + }; + let response_plaintext_bytes = mc_util_serial::encode(&client_query_response); + let response = + self.ake + .client_encrypt(&channel_id, &sealed_query.aad, &response_plaintext_bytes)?; + + Ok(response) + } + fn handle_key_image_store_request( - &self, - _: EnclaveMessage - ) -> Result> { - todo!() + &self, + _: EnclaveMessage, + ) -> Result> { + todo!() } } diff --git a/fog/ledger/enclave/src/lib.rs b/fog/ledger/enclave/src/lib.rs index e29d2170ee..ed3ccdefb2 100644 --- a/fog/ledger/enclave/src/lib.rs +++ b/fog/ledger/enclave/src/lib.rs @@ -14,7 +14,9 @@ pub use mc_fog_ledger_enclave_api::{ use mc_attest_core::{ IasNonce, Quote, QuoteNonce, Report, SgxError, TargetInfo, VerificationReport, }; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; use mc_attest_verifier::DEBUG_ENCLAVE; use mc_common::{logger::Logger, ResponderId}; use mc_crypto_keys::X25519Public; @@ -25,7 +27,7 @@ use mc_sgx_types::{ sgx_attributes_t, sgx_enclave_id_t, sgx_launch_token_t, sgx_misc_attribute_t, sgx_status_t, }; use mc_sgx_urts::SgxEnclave; -use std::{path, result::Result as StdResult, sync::Arc}; +use std::{collections::BTreeMap, path, result::Result as StdResult, sync::Arc}; /// The default filename of the fog ledger's SGX enclave binary. pub const ENCLAVE_FILE: &str = "libledger-enclave.signed.so"; @@ -189,9 +191,16 @@ impl LedgerEnclave for LedgerSgxEnclave { } // Router/store system. - fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) -> Result { - mc_sgx_debug::eprintln!("Called connect_to_key_image_store(ledger_store_id: {})", ledger_store_id); - let inbuf = mc_util_serial::serialize(&EnclaveCall::ConnectToKeyImageStore(ledger_store_id))?; + fn connect_to_key_image_store( + &self, + ledger_store_id: ResponderId, + ) -> Result { + mc_sgx_debug::eprintln!( + "Called connect_to_key_image_store(ledger_store_id: {})", + ledger_store_id + ); + let inbuf = + mc_util_serial::serialize(&EnclaveCall::ConnectToKeyImageStore(ledger_store_id))?; let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } @@ -203,17 +212,49 @@ impl LedgerEnclave for LedgerSgxEnclave { ledger_store_auth_response: ClientAuthResponse, ) -> Result<()> { mc_sgx_debug::eprintln!("Called finish_connecting_to_key_image_store(ledger_store_id: {}, ledger_store_auth_response: {:?})", ledger_store_id, ledger_store_auth_response); - let inbuf = mc_util_serial::serialize(&EnclaveCall::FinishConnectingToKeyImageStore(ledger_store_id, ledger_store_auth_response))?; + + let inbuf = mc_util_serial::serialize(&EnclaveCall::FinishConnectingToKeyImageStore( + ledger_store_id, + ledger_store_auth_response, + ))?; + let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } - fn create_key_image_store_query( + fn decrypt_and_seal_query( &self, client_query: EnclaveMessage, + ) -> Result { + mc_sgx_debug::eprintln!( + "Called decrypt_and_seal_query(..) - the router is handling a message from the client" + ); + let inbuf = mc_util_serial::serialize(&EnclaveCall::DecryptAndSealQuery(client_query))?; + let outbuf = self.enclave_call(&inbuf)?; + mc_util_serial::deserialize(&outbuf[..])? + } + + fn create_multi_key_image_store_query_data( + &self, + sealed_query: SealedClientMessage, ) -> Result>> { - mc_sgx_debug::eprintln!("Called create_key_image_store_query(..) - the router is handling a message from the client"); - let inbuf = mc_util_serial::serialize(&EnclaveCall::CreateKeyImageStoreQuery(client_query))?; + mc_sgx_debug::eprintln!("Called create_multi_key_image_store_query_data(..) - the router is handling a message from the client"); + let inbuf = mc_util_serial::serialize(&EnclaveCall::CreateMultiKeyImageStoreQueryData( + sealed_query, + ))?; + let outbuf = self.enclave_call(&inbuf)?; + mc_util_serial::deserialize(&outbuf[..])? + } + + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result> { + let inbuf = mc_util_serial::serialize(&EnclaveCall::CollateQueryResponses( + sealed_query, + shard_query_responses, + ))?; let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } @@ -223,7 +264,10 @@ impl LedgerEnclave for LedgerSgxEnclave { router_query: EnclaveMessage, ) -> Result> { mc_sgx_debug::eprintln!("Called handle_key_image_store_request(..) - the store is handling a message from the router."); - let inbuf = mc_util_serial::serialize(&EnclaveCall::HandleKeyImageStoreRequest(router_query))?; + + let inbuf = + mc_util_serial::serialize(&EnclaveCall::HandleKeyImageStoreRequest(router_query))?; + let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } diff --git a/fog/ledger/enclave/trusted/Cargo.lock b/fog/ledger/enclave/trusted/Cargo.lock index ac316e2279..477ae5d758 100644 --- a/fog/ledger/enclave/trusted/Cargo.lock +++ b/fog/ledger/enclave/trusted/Cargo.lock @@ -795,6 +795,29 @@ dependencies = [ "serde", ] +[[package]] +name = "mc-blockchain-types" +version = "1.3.0-pre0" +dependencies = [ + "displaydoc", + "hex_fmt", + "mc-account-keys", + "mc-attest-verifier-types", + "mc-common", + "mc-consensus-scp-types", + "mc-crypto-digestible", + "mc-crypto-digestible-signature", + "mc-crypto-keys", + "mc-crypto-ring-signature", + "mc-transaction-core", + "mc-transaction-types", + "mc-util-from-random", + "mc-util-repr-bytes", + "prost", + "serde", + "zeroize", +] + [[package]] name = "mc-common" version = "1.3.0-pre0" @@ -816,6 +839,18 @@ dependencies = [ "slog", ] +[[package]] +name = "mc-consensus-scp-types" +version = "1.3.0-pre0" +dependencies = [ + "mc-common", + "mc-crypto-digestible", + "mc-crypto-keys", + "mc-util-from-random", + "prost", + "serde", +] + [[package]] name = "mc-crypto-ake-enclave" version = "1.3.0-pre0" @@ -1059,6 +1094,7 @@ dependencies = [ "aligned-cmov", "mc-attest-core", "mc-attest-enclave-api", + "mc-blockchain-types", "mc-common", "mc-crypto-ake-enclave", "mc-crypto-keys", diff --git a/fog/ledger/enclave/trusted/src/lib.rs b/fog/ledger/enclave/trusted/src/lib.rs index 641a8446aa..9ad9aaefd9 100644 --- a/fog/ledger/enclave/trusted/src/lib.rs +++ b/fog/ledger/enclave/trusted/src/lib.rs @@ -64,16 +64,27 @@ pub fn ecall_dispatcher(inbuf: &[u8]) -> Result, sgx_status_t> { // Router / Store system // Router-side - EnclaveCall::ConnectToKeyImageStore(responder_id) => serialize(&ENCLAVE.connect_to_key_image_store(responder_id)), - EnclaveCall::FinishConnectingToKeyImageStore( - responder_id, - client_auth_response, - ) => serialize(&ENCLAVE.finish_connecting_to_key_image_store(responder_id, client_auth_response)), - EnclaveCall::CreateKeyImageStoreQuery(msg) => - serialize(&ENCLAVE.create_key_image_store_query(msg)), + EnclaveCall::ConnectToKeyImageStore(responder_id) => { + serialize(&ENCLAVE.connect_to_key_image_store(responder_id)) + } + EnclaveCall::FinishConnectingToKeyImageStore(responder_id, client_auth_response) => { + serialize( + &ENCLAVE.finish_connecting_to_key_image_store(responder_id, client_auth_response), + ) + } + EnclaveCall::DecryptAndSealQuery(client_query) => { + serialize(&ENCLAVE.decrypt_and_seal_query(client_query)) + } + EnclaveCall::CreateMultiKeyImageStoreQueryData(msg) => { + serialize(&ENCLAVE.create_multi_key_image_store_query_data(msg)) + } + EnclaveCall::CollateQueryResponses(sealed_query, shard_query_responses) => { + serialize(&ENCLAVE.collate_shard_query_responses(sealed_query, shard_query_responses)) + } // Store-side - EnclaveCall::HandleKeyImageStoreRequest(msg) => - serialize(&ENCLAVE.handle_key_image_store_request(msg)), + EnclaveCall::HandleKeyImageStoreRequest(msg) => { + serialize(&ENCLAVE.handle_key_image_store_request(msg)) + } } .or(Err(sgx_status_t::SGX_ERROR_UNEXPECTED)) } diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 30f9885b04..d651f415e4 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -15,7 +15,7 @@ path = "src/bin/main.rs" [[bin]] name = "ledger_router_server" -path = "src/bin/router.rs" +path = "src/bin/key_image_router.rs" [dependencies] mc-attest-api = { path = "../../../attest/api" } diff --git a/fog/ledger/server/src/bin/key_image_router.rs b/fog/ledger/server/src/bin/key_image_router.rs index 9106ae33f6..a7424dad98 100644 --- a/fog/ledger/server/src/bin/key_image_router.rs +++ b/fog/ledger/server/src/bin/key_image_router.rs @@ -1,13 +1,15 @@ -use std::{env, sync::Arc, str::FromStr}; +// Copyright (c) 2018-2022 The MobileCoin Foundation +use std::{env, str::FromStr, sync::Arc}; + +use clap::Parser; use grpcio::ChannelBuilder; use mc_common::logger::log; use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; -use mc_fog_ledger_enclave::{ENCLAVE_FILE, LedgerSgxEnclave}; -use mc_fog_ledger_server::LedgerRouterConfig; -use mc_fog_ledger_server::KeyImageRouterServer; -use clap::Parser; -use mc_fog_uri::{KeyImageStoreUri, KeyImageStoreScheme}; +use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; +use mc_fog_ledger_server::{KeyImageRouterServer, LedgerRouterConfig}; +use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; + use mc_util_grpc::ConnectionUriGrpcioChannel; use mc_util_uri::UriScheme; @@ -32,7 +34,7 @@ fn main() { config.omap_capacity, logger.clone(), ); - + let mut ledger_store_grpc_clients: Vec = Vec::new(); let grpc_env = Arc::new( grpcio::EnvBuilder::new() @@ -60,4 +62,4 @@ fn main() { loop { std::thread::sleep(std::time::Duration::from_millis(1000)); } -} \ No newline at end of file +} diff --git a/fog/ledger/server/src/config.rs b/fog/ledger/server/src/config.rs index 2b33492644..daaa041b85 100644 --- a/fog/ledger/server/src/config.rs +++ b/fog/ledger/server/src/config.rs @@ -11,7 +11,7 @@ use mc_fog_uri::{FogLedgerUri, KeyImageRouterUri, KeyImageStoreUri}; use mc_util_parse::parse_duration_in_seconds; use mc_util_uri::AdminUri; use serde::Serialize; -use std::{path::PathBuf, time::Duration, str::FromStr}; +use std::{path::PathBuf, str::FromStr, time::Duration}; /// Configuration parameters for the ledger server #[derive(Clone, Parser, Serialize)] @@ -79,8 +79,7 @@ pub struct LedgerServerConfig { /// A Fog Server can either fulfill client requests directly or fulfill Fog /// Ledger Router requests, and these types of servers use different URLs. -/// -/// TODO - This is almost identical to Fog View's implementation of this +/// TODO - This is almost identical to Fog View's implementation of this /// combine it later? #[derive(Clone, Serialize)] pub enum KeyImageClientListenUri { @@ -100,7 +99,10 @@ impl FromStr for KeyImageClientListenUri { return Ok(KeyImageClientListenUri::Store(ledger_store_uri)); } - Err(format!("Incorrect KeyImageClientListenUri string: {}.", input)) + Err(format!( + "Incorrect KeyImageClientListenUri string: {}.", + input + )) } } diff --git a/fog/ledger/server/src/error.rs b/fog/ledger/server/src/error.rs index dbeeefb014..521e49879a 100644 --- a/fog/ledger/server/src/error.rs +++ b/fog/ledger/server/src/error.rs @@ -1,3 +1,5 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + use displaydoc::Display; use grpcio::RpcStatus; use mc_common::logger::Logger; @@ -5,7 +7,6 @@ use mc_fog_ledger_enclave_api::Error as LedgerEnclaveError; use mc_sgx_report_cache_untrusted::Error as ReportCacheError; use mc_util_grpc::{rpc_internal_error, rpc_permissions_error}; - #[derive(Debug, Display)] pub enum RouterServerError { /// Error related to contacting Fog Ledger Store: {0} @@ -32,7 +33,12 @@ impl From for RouterServerError { } } -#[allow(dead_code)] //FIXME +impl From for RouterServerError { + fn from(src: mc_util_uri::UriConversionError) -> Self { + RouterServerError::LedgerStoreError(format!("{}", src)) + } +} + pub fn router_server_err_to_rpc_status( context: &str, src: RouterServerError, @@ -54,7 +60,7 @@ impl From for RouterServerError { } } -#[allow(dead_code)] // FIXME when the ledger router is more than just a skeleton. +#[allow(dead_code)] // FIXME when the ledger router is more than just a skeleton. #[derive(Debug, Display)] pub enum LedgerServerError { /// Ledger Enclave error: {0} diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs index 608bd343c9..1eb71d8c4a 100644 --- a/fog/ledger/server/src/key_image_router_server.rs +++ b/fog/ledger/server/src/key_image_router_server.rs @@ -7,9 +7,9 @@ use mc_common::logger::{log, Logger}; use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::ConnectionUri; -use mc_util_grpc::{ReadinessIndicator, ConnectionUriGrpcioServer}; +use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; -use crate::{key_image_router_service::KeyImageRouterService, config::LedgerRouterConfig}; +use crate::{config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService}; #[allow(dead_code)] // FIXME pub struct KeyImageRouterServer { @@ -20,7 +20,7 @@ pub struct KeyImageRouterServer { impl KeyImageRouterServer { /// Creates a new ledger router server instance #[allow(dead_code)] // FIXME - pub fn new ( + pub fn new( config: LedgerRouterConfig, enclave: E, shards: Vec, @@ -39,14 +39,12 @@ impl KeyImageRouterServer { // Health check service - will be used in both cases let health_service = - mc_util_grpc::HealthService::new( - Some(readiness_indicator.into()), logger.clone() - ).into_service(); + mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) + .into_service(); match config.client_listen_uri { // Router server crate::config::KeyImageClientListenUri::ClientFacing(ledger_router_uri) => { - // Init ledger router service. let ledger_router_service = ledger_grpc::create_ledger_api( KeyImageRouterService::new(enclave, shards, logger.clone()), @@ -63,15 +61,14 @@ impl KeyImageRouterServer { .register_service(ledger_router_service) .register_service(health_service) .bind_using_uri(&ledger_router_uri, logger.clone()); - let server = server_builder.build().unwrap(); - + Self { server, logger } - }, - // Store server. + } + // Store server. crate::config::KeyImageClientListenUri::Store(_ledger_store_uri) => { todo!() - }, + } } } diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs index 3da6a18790..2a2e3be33a 100644 --- a/fog/ledger/server/src/key_image_router_service.rs +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -1,11 +1,13 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + use std::sync::Arc; -use futures::{TryFutureExt, FutureExt}; +use futures::{FutureExt, TryFutureExt}; use grpcio::{DuplexSink, RequestStream, RpcContext}; use mc_common::logger::{log, Logger}; use mc_fog_api::{ - ledger_grpc::{LedgerApi, self}, - ledger::{LedgerRequest, LedgerResponse}, + ledger::{LedgerRequest, LedgerResponse}, + ledger_grpc::{self, LedgerApi}, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_util_grpc::rpc_logger; @@ -28,7 +30,11 @@ impl KeyImageRouterService { /// Creates a new LedgerRouterService that can be used by a gRPC server to /// fulfill gRPC requests. #[allow(dead_code)] // FIXME - pub fn new(enclave: E, shards: Vec, logger: Logger) -> Self { + pub fn new( + enclave: E, + shards: Vec, + logger: Logger, + ) -> Self { let shards = shards.into_iter().map(Arc::new).collect(); Self { enclave, @@ -52,8 +58,10 @@ where log::info!(self.logger, "Request received in request fn"); let _timer = SVC_COUNTERS.req(&ctx); mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { - log::warn!(self.logger, "Streaming GRPC Ledger API only partially implemented."); - + log::warn!( + self.logger, + "Streaming GRPC Ledger API only partially implemented." + ); let logger = logger.clone(); let future = router_handlers::handle_requests( @@ -64,10 +72,10 @@ where logger.clone(), ) .map_err(move |err: grpcio::Error| log::error!(&logger, "failed to reply: {}", err)) - // TODO: Do more with the error than just push it to the log. + // TODO: Do more with the error than just push it to the log. .map(|_| ()); ctx.spawn(future) }); } -} \ No newline at end of file +} diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index 02a58da459..dcac2797ae 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -7,9 +7,9 @@ use mc_attest_api::{ }; use mc_blockchain_types::MAX_BLOCK_VERSION; use mc_common::logger::{log, Logger}; -use mc_fog_api::{ - ledger_grpc::{FogKeyImageApi, KeyImageStoreApi}, +use mc_fog_api::{ ledger::{MultiKeyImageStoreRequest, MultiKeyImageStoreResponse}, + ledger_grpc::{FogKeyImageApi, KeyImageStoreApi}, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_ledger_enclave_api::{Error as EnclaveError, UntrustedKeyImageQueryResponse}; @@ -183,12 +183,22 @@ impl FogKeyImageApi for KeyImageServic impl KeyImageStoreApi for KeyImageService { #[allow(unused_variables)] //FIXME - fn auth(&mut self, ctx: grpcio::RpcContext, req: AuthMessage, sink: grpcio::UnarySink) { + fn auth( + &mut self, + ctx: grpcio::RpcContext, + req: AuthMessage, + sink: grpcio::UnarySink, + ) { todo!() } #[allow(unused_variables)] //FIXME - fn multi_key_image_store_query(&mut self, ctx: grpcio::RpcContext, req: MultiKeyImageStoreRequest, sink: grpcio::UnarySink) { + fn multi_key_image_store_query( + &mut self, + ctx: grpcio::RpcContext, + req: MultiKeyImageStoreRequest, + sink: grpcio::UnarySink, + ) { todo!() } -} \ No newline at end of file +} diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index 70694bb27a..0dfa1982e0 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -11,9 +11,10 @@ mod router_handlers; mod server; mod untrusted_tx_out_service; -//Router & store system. KeyImageService can function as a Store but the router is implemented as a different GRPC server struct. -mod key_image_router_service; +//Router & store system. KeyImageService can function as a Store but the router +// is implemented as a different GRPC server struct. mod key_image_router_server; +mod key_image_router_service; pub use block_service::BlockService; pub use config::LedgerServerConfig; @@ -23,4 +24,4 @@ pub use server::LedgerServer; pub use untrusted_tx_out_service::UntrustedTxOutService; pub use config::LedgerRouterConfig; -pub use key_image_router_server::KeyImageRouterServer; \ No newline at end of file +pub use key_image_router_server::KeyImageRouterServer; diff --git a/fog/ledger/server/src/router_handlers.rs b/fog/ledger/server/src/router_handlers.rs index 411f488cc9..2fe5a7c170 100644 --- a/fog/ledger/server/src/router_handlers.rs +++ b/fog/ledger/server/src/router_handlers.rs @@ -1,20 +1,23 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use crate::{ - error::{router_server_err_to_rpc_status, RouterServerError}, -}; -use futures::{future::try_join_all, TryStreamExt, SinkExt}; -use grpcio::{DuplexSink, RequestStream, RpcStatus, WriteFlags}; +use crate::error::{router_server_err_to_rpc_status, RouterServerError}; +use futures::{future::try_join_all, SinkExt, TryStreamExt}; +use grpcio::{ChannelBuilder, DuplexSink, RequestStream, RpcStatus, WriteFlags}; use mc_attest_api::attest; -use mc_common::{logger::Logger}; +use mc_attest_enclave_api::{ClientSession, EnclaveMessage}; +use mc_common::{logger::Logger, ResponderId}; use mc_fog_api::{ - ledger::{LedgerRequest, LedgerResponse, MultiKeyImageStoreRequest, MultiKeyImageStoreResponse}, ledger_grpc::KeyImageStoreApiClient, + ledger::{ + LedgerRequest, LedgerResponse, MultiKeyImageStoreRequest, MultiKeyImageStoreResponse, + MultiKeyImageStoreResponseStatus, + }, + ledger_grpc::KeyImageStoreApiClient, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::KeyImageStoreUri; +use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; //use mc_fog_ledger_enclave_api::LedgerEnclaveProxy; -use mc_util_grpc::{rpc_invalid_arg_error}; -use std::{str::FromStr, sync::Arc}; +use mc_util_grpc::{rpc_invalid_arg_error, ConnectionUriGrpcioChannel}; +use std::{collections::BTreeMap, str::FromStr, sync::Arc}; #[allow(dead_code)] //FIXME const RETRY_COUNT: usize = 3; @@ -62,9 +65,16 @@ where { if request.has_auth() { handle_auth_request(enclave, request.take_auth(), logger) - } else if request.has_check_key_images() { - handle_query_request(request.take_check_key_images(), enclave, shard_clients, logger).await - // TODO: Handle other cases here as they are added, such as the merkele proof service. + } else if request.has_check_key_images() + handle_query_request( + request.take_check_key_images(), + enclave, + shard_clients, + logger, + ) + .await + // TODO: Handle other cases here as they are added, such as the merkele + // proof service. } else { let rpc_status = rpc_invalid_arg_error( "Inavlid LedgerRequest request", @@ -75,7 +85,8 @@ where } } -/// The result of processing the MultiLedgerStoreQueryResponse from each Fog Ledger Shard. +/// The result of processing the MultiLedgerStoreQueryResponse from each Fog +/// Ledger Shard. pub struct ProcessedShardResponseData { /// gRPC clients for Shards that need to be retried for a successful /// response. @@ -87,14 +98,14 @@ pub struct ProcessedShardResponseData { pub store_uris_for_authentication: Vec, /// New, successfully processed query responses. - pub new_query_responses: Vec, + pub new_query_responses: Vec<(ResponderId, attest::Message)>, } impl ProcessedShardResponseData { pub fn new( shard_clients_for_retry: Vec>, store_uris_for_authentication: Vec, - new_query_responses: Vec, + new_query_responses: Vec<(ResponderId, attest::Message)>, ) -> Self { ProcessedShardResponseData { shard_clients_for_retry, @@ -116,13 +127,18 @@ pub fn process_shard_responses( // We did not receive a query_response for this shard.Therefore, we need to: // (a) retry the query // (b) authenticate with the Ledger Store that returned the decryption_error - if response.has_decryption_error() { - shard_clients_for_retry.push(shard_client); - let store_uri = - KeyImageStoreUri::from_str(&response.get_decryption_error().store_uri)?; - store_uris_for_authentication.push(store_uri); - } else { - new_query_responses.push(response.take_query_response()); + let store_uri = KeyImageStoreUri::from_str(response.get_fog_ledger_store_uri())?; + match response.get_status() { + MultiKeyImageStoreResponseStatus::SUCCESS => { + let store_responder_id = store_uri.responder_id()?; + new_query_responses.push((store_responder_id, response.take_query_response())); + } + MultiKeyImageStoreResponseStatus::AUTHENTICATION_ERROR => { + shard_clients_for_retry.push(shard_client); + store_uris_for_authentication.push(store_uri); + } + // This call will be retried as part of the larger retry logic + MultiKeyImageStoreResponseStatus::NOT_READY => (), } } @@ -151,7 +167,7 @@ where Ok(response) } -#[allow(unused_variables)] // FIXME when enclave code is set up. +#[allow(unused_variables)] // FIXME when enclave code is set up. /// Handles a client's query request. async fn handle_query_request( query: attest::Message, @@ -162,13 +178,29 @@ async fn handle_query_request( where E: LedgerEnclaveProxy, { - let mut query_responses: Vec = Vec::with_capacity(shard_clients.len()); + let mut query_responses: BTreeMap> = BTreeMap::new(); let mut shard_clients = shard_clients.clone(); - // TODO: use retry crate? - for _ in 0..RETRY_COUNT { - /* + let sealed_query = enclave + .decrypt_and_seal_query(query.into()) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: internal encryption error", + err.into(), + logger.clone(), + ) + })?; + + // The retry logic here is: + // Set retries remaining to RETRY_COUNT + // Send query and process responses + // If there's a response from every shard, we're done + // If there's a new store, repeat + // If there's no new store and we don't have enough responses, decrement + // RETRY_COUNT and loop + let mut remaining_retries = RETRY_COUNT; + while remaining_retries > 0 { let multi_ledger_store_query_request = enclave - .create_multi_key_image_store_query_data(query.clone().into()) + .create_multi_key_image_store_query_data(sealed_query.clone()) .map_err(|err| { router_server_err_to_rpc_status( "Query: internal encryption error", @@ -176,10 +208,9 @@ where logger.clone(), ) })? - .into();*/ - let test_request = MultiKeyImageStoreRequest::default(); + .into(); let clients_and_responses = - route_query(&test_request, shard_clients.clone()) + route_query(&multi_ledger_store_query_request, shard_clients.clone()) .await .map_err(|err| { router_server_err_to_rpc_status( @@ -189,36 +220,62 @@ where ) })?; - let mut processed_shard_response_data = process_shard_responses( - clients_and_responses, - ) - .map_err(|err| { - router_server_err_to_rpc_status( - "Query: internal query response processing", - err, - logger.clone(), - ) - })?; + let processed_shard_response_data = process_shard_responses(clients_and_responses) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: internal query response processing", + err, + logger.clone(), + ) + })?; - query_responses.append(&mut processed_shard_response_data.new_query_responses); - shard_clients = processed_shard_response_data.shard_clients_for_retry; - if shard_clients.is_empty() { + for (store_responder_id, new_query_response) in processed_shard_response_data + .new_query_responses + .into_iter() + { + query_responses.insert(store_responder_id, new_query_response.into()); + } + + if query_responses.len() >= shard_clients.len() { break; } - /* TODO pending ledger router code enclave-side. - - authenticate_ledger_stores( - enclave.clone(), - processed_shard_response_data.store_uris_for_authentication, + shard_clients = processed_shard_response_data.shard_clients_for_retry; + if !shard_clients.is_empty() { + authenticate_ledger_stores( + enclave.clone(), + processed_shard_response_data.store_uris_for_authentication, + logger.clone(), + ) + .await?; + } else { + remaining_retries -= 1; + } + } + + if remaining_retries == 0 { + return Err(router_server_err_to_rpc_status( + "Query: timed out connecting to key image stores", + RouterServerError::LedgerStoreError(format!( + "Received {} responses which failed to advance the MultiKeyImageStoreRequest", + RETRY_COUNT + )), logger.clone(), - ) - .await?;*/ + )); } - // TODO: Collate the query_responses into one response for the client. Make an - // enclave method for this. - let response = LedgerResponse::new(); + let query_response = enclave + .collate_shard_query_responses(sealed_query, query_responses) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: shard response collation", + RouterServerError::Enclave(err), + logger.clone(), + ) + })?; + + let mut response = LedgerResponse::new(); + response.set_check_key_image_response(query_response.into()); Ok(response) } @@ -244,16 +301,12 @@ async fn query_shard( Ok((shard_client, response)) } - - -/* TODO pending ledger router code enclave-side. - // Authenticates Fog Ledger Stores that have previously not been authenticated. async fn authenticate_ledger_stores( enclave: E, - ledger_store_uris: Vec, + ledger_store_uris: Vec, logger: Logger, -) -> Result<(), RpcStatus> { +) -> Result, RpcStatus> { let pending_auth_requests = ledger_store_uris .into_iter() .map(|store_uri| authenticate_ledger_store(enclave.clone(), store_uri, logger.clone())); @@ -271,25 +324,26 @@ async fn authenticate_ledger_stores( // Authenticates a Fog Ledger Store that has previously not been authenticated. async fn authenticate_ledger_store( enclave: E, - ledger_store_url: LedgerStoreUri, + ledger_store_url: KeyImageStoreUri, logger: Logger, ) -> Result<(), RouterServerError> { let ledger_store_id = ResponderId::from_str(&ledger_store_url.to_string())?; - let client_auth_request = enclave.ledger_store_init(ledger_store_id.clone())?; + let client_auth_request = enclave.connect_to_key_image_store(ledger_store_id.clone())?; let grpc_env = Arc::new( grpcio::EnvBuilder::new() .name_prefix("authenticate-ledger-store".to_string()) .build(), ); let ledger_store_client = KeyImageStoreApiClient::new( - ChannelBuilder::default_channel_builder(grpc_env).connect_to_uri(&ledger_store_url, &logger), + ChannelBuilder::default_channel_builder(grpc_env) + .connect_to_uri(&ledger_store_url, &logger), ); let auth_unary_receiver = ledger_store_client.auth_async(&client_auth_request.into())?; let auth_response = auth_unary_receiver.await?; - let result = enclave.ledger_store_connect(ledger_store_id, auth_response.into())?; + let result = + enclave.finish_connecting_to_key_image_store(ledger_store_id, auth_response.into())?; Ok(result) } -*/ \ No newline at end of file diff --git a/fog/ledger/test_infra/src/lib.rs b/fog/ledger/test_infra/src/lib.rs index ca5970b37f..7fc53d2073 100644 --- a/fog/ledger/test_infra/src/lib.rs +++ b/fog/ledger/test_infra/src/lib.rs @@ -85,7 +85,10 @@ impl LedgerEnclave for MockEnclave { unimplemented!() } - fn connect_to_key_image_store(&self, _ledger_store_id: ResponderId) -> EnclaveResult { + fn connect_to_key_image_store( + &self, + _ledger_store_id: ResponderId, + ) -> EnclaveResult { unimplemented!() } @@ -105,7 +108,7 @@ impl LedgerEnclave for MockEnclave { } fn handle_key_image_store_request( - &self, + &self, _router_query: EnclaveMessage, ) -> EnclaveResult> { unimplemented!() diff --git a/fog/view/connection/src/bin/router_client.rs b/fog/view/connection/src/bin/router_client.rs deleted file mode 100644 index c095e750b1..0000000000 --- a/fog/view/connection/src/bin/router_client.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -//! A simple binary that creates a Fog View Router Client and sends off -//! requests to the Fog View Router server. - -use mc_fog_uri::FogViewRouterUri; -use mc_fog_view_connection::fog_view_router_client::FogViewRouterGrpcClient; -use std::{str::FromStr, sync::Arc}; - -#[tokio::main(flavor = "multi_thread")] -async fn main() -> Result<(), grpcio::Error> { - let fog_view_router_uri = FogViewRouterUri::from_str("insecure-fog-view-router://127.0.0.1/") - .expect("failed to connect to fog view router uri"); - let env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("Main-RPC".to_string()) - .build(), - ); - let (logger, _global_logger_guard) = - mc_common::logger::create_app_logger(mc_common::logger::o!()); - - let fog_view_router_client = - FogViewRouterGrpcClient::new(fog_view_router_uri.clone(), env.clone(), logger.clone()); - - fog_view_router_client.request().await -} diff --git a/fog/view/enclave/trusted/Cargo.lock b/fog/view/enclave/trusted/Cargo.lock index 4f0c8d527a..d279dd1932 100644 --- a/fog/view/enclave/trusted/Cargo.lock +++ b/fog/view/enclave/trusted/Cargo.lock @@ -1135,6 +1135,7 @@ dependencies = [ "mc-attest-core", "mc-attest-enclave-api", "mc-common", + "mc-crypto-ake-enclave", "mc-crypto-keys", "mc-crypto-noise", "mc-fog-recovery-db-iface", diff --git a/fog/view/server/src/config.rs b/fog/view/server/src/config.rs index 815d73f4bd..c5f8724b1f 100644 --- a/fog/view/server/src/config.rs +++ b/fog/view/server/src/config.rs @@ -205,6 +205,14 @@ pub struct FogViewRouterConfig { #[clap(long, env = "MC_CLIENT_LISTEN_URI")] pub client_listen_uri: FogViewRouterUri, + /// PEM-formatted keypair to send with an Attestation Request. + #[clap(long, env = "MC_IAS_API_KEY")] + pub ias_api_key: String, + + /// The IAS SPID to use when getting a quote + #[clap(long, env = "MC_IAS_SPID")] + pub ias_spid: ProviderId, + // TODO: Add shard uris which are of type Vec. /// The capacity to build the OMAP (ORAM hash table) with. /// About 75% of this capacity can be used.