diff --git a/Cargo.lock b/Cargo.lock index 92b24a0b2b..2aee910677 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3701,6 +3701,7 @@ dependencies = [ "aligned-cmov", "mc-attest-core", "mc-attest-enclave-api", + "mc-blockchain-types", "mc-common", "mc-crypto-ake-enclave", "mc-crypto-keys", @@ -4296,13 +4297,17 @@ dependencies = [ name = "mc-fog-view-connection" version = "1.3.0-pre0" dependencies = [ + "aes-gcm", "futures", "grpcio", + "mc-attest-ake", "mc-attest-api", "mc-attest-core", "mc-attest-verifier", "mc-common", "mc-crypto-keys", + "mc-crypto-noise", + "mc-crypto-rand", "mc-fog-api", "mc-fog-enclave-connection", "mc-fog-types", @@ -4311,7 +4316,9 @@ dependencies = [ "mc-util-grpc", "mc-util-serial", "mc-util-telemetry", + "mc-util-uri", "retry", + "sha2 0.10.2", "tokio", ] @@ -4325,6 +4332,7 @@ dependencies = [ "mc-attest-enclave-api", "mc-attest-verifier", "mc-common", + "mc-crypto-ake-enclave", "mc-crypto-keys", "mc-enclave-boundary", "mc-fog-ocall-oram-storage-edl", @@ -4358,6 +4366,7 @@ dependencies = [ "mc-attest-core", "mc-attest-enclave-api", "mc-common", + "mc-crypto-ake-enclave", "mc-crypto-keys", "mc-crypto-noise", "mc-fog-recovery-db-iface", @@ -4383,6 +4392,7 @@ version = "1.3.0-pre0" dependencies = [ "aes-gcm", "aligned-cmov", + "itertools", "mc-attest-ake", "mc-attest-core", "mc-attest-enclave-api", @@ -4468,6 +4478,7 @@ dependencies = [ "lazy_static", "mc-attest-api", "mc-attest-core", + "mc-attest-enclave-api", "mc-attest-net", "mc-attest-verifier", "mc-blockchain-types", diff --git a/attest/ake/src/event.rs b/attest/ake/src/event.rs index 39987b04c8..c38277522b 100644 --- a/attest/ake/src/event.rs +++ b/attest/ake/src/event.rs @@ -351,3 +351,37 @@ impl MealyInput for Ciphertext<'_, '_> {} /// Our outputs may be simple vectors for the proto-inside-grpc use case. impl MealyOutput for Vec {} + +/// A type similar to [`aead::Payload`] used to distinguish writer inputs from +/// outputs when there's an explicit nonce. +pub struct NoncePlaintext<'aad, 'msg> { + pub aad: &'aad [u8], + pub msg: &'msg [u8], + pub nonce: u64, +} + +impl<'aad, 'msg> NoncePlaintext<'aad, 'msg> { + pub fn new(aad: &'aad [u8], msg: &'msg [u8], nonce: u64) -> Self { + Self { aad, msg, nonce } + } +} + +/// Plaintext may be provided to an FST for encryption into a vector +impl MealyInput for NoncePlaintext<'_, '_> {} + +/// A type similar to [`aead::Payload`] used to distinguish reader inputs from +/// outputs when there's an explicit nonce. +pub struct NonceCiphertext<'aad, 'msg> { + pub aad: &'aad [u8], + pub msg: &'msg [u8], + pub nonce: u64, +} + +impl<'aad, 'msg> NonceCiphertext<'aad, 'msg> { + pub fn new(aad: &'aad [u8], msg: &'msg [u8], nonce: u64) -> Self { + Self { aad, msg, nonce } + } +} + +/// Plaintext may be provided to an FST for encryption into a vector +impl MealyInput for NonceCiphertext<'_, '_> {} diff --git a/attest/ake/src/shared.rs b/attest/ake/src/shared.rs index 37676bae6d..2c8e19b36a 100644 --- a/attest/ake/src/shared.rs +++ b/attest/ake/src/shared.rs @@ -3,7 +3,7 @@ //! Common transitions between initiator and responder. use crate::{ - event::{Ciphertext, Plaintext}, + event::{Ciphertext, NonceCiphertext, NoncePlaintext, Plaintext}, mealy::Transition, state::Ready, }; @@ -46,3 +46,39 @@ where Ok((retval, ciphertext)) } } + +/// Ready + NonceCiphertext => Ready + Vec +impl Transition, NonceCiphertext<'_, '_>, Vec> for Ready +where + Cipher: NoiseCipher, +{ + type Error = CipherError; + + fn try_next( + self, + _csprng: &mut R, + input: NonceCiphertext<'_, '_>, + ) -> Result<(Ready, Vec), Self::Error> { + let mut retval = self; + let plaintext = retval.decrypt_with_nonce(input.aad, input.msg, input.nonce)?; + Ok((retval, plaintext)) + } +} + +/// Ready + NoncePlaintext => Ready + Vec +impl Transition, NoncePlaintext<'_, '_>, Vec> for Ready +where + Cipher: NoiseCipher, +{ + type Error = CipherError; + + fn try_next( + self, + _csprng: &mut R, + input: NoncePlaintext<'_, '_>, + ) -> Result<(Ready, Vec), Self::Error> { + let mut retval = self; + let ciphertext = retval.encrypt_with_nonce(input.aad, input.msg, input.nonce)?; + Ok((retval, ciphertext)) + } +} diff --git a/attest/ake/src/state.rs b/attest/ake/src/state.rs index 370afadfde..4d9e7ed28c 100644 --- a/attest/ake/src/state.rs +++ b/attest/ake/src/state.rs @@ -74,6 +74,7 @@ where pub fn binding(&self) -> &[u8] { self.binding.as_ref() } + /// Using the writer cipher, encrypt the given plaintext. pub fn encrypt(&mut self, aad: &[u8], plaintext: &[u8]) -> Result, CipherError> { self.writer.encrypt_with_ad(aad, plaintext) @@ -83,6 +84,28 @@ where pub fn decrypt(&mut self, aad: &[u8], ciphertext: &[u8]) -> Result, CipherError> { self.reader.decrypt_with_ad(aad, ciphertext) } + + /// Using the writer cipher, encrypt the given plaintext for the nonce. + pub fn encrypt_with_nonce( + &mut self, + aad: &[u8], + plaintext: &[u8], + nonce: u64, + ) -> Result, CipherError> { + self.writer.set_nonce(nonce); + self.encrypt(aad, plaintext) + } + + /// Using the reader cipher, decrypt the provided ciphertext for the nonce. + pub fn decrypt_with_nonce( + &mut self, + aad: &[u8], + ciphertext: &[u8], + nonce: u64, + ) -> Result, CipherError> { + self.reader.set_nonce(nonce); + self.decrypt(aad, ciphertext) + } } impl State for Ready where Cipher: NoiseCipher {} diff --git a/attest/api/proto/attest.proto b/attest/api/proto/attest.proto index 799f91c3fb..1461774086 100644 --- a/attest/api/proto/attest.proto +++ b/attest/api/proto/attest.proto @@ -48,3 +48,20 @@ message Message { /// for use in the enclave. bytes data = 3; } + +/// An AEAD message with an explicit nonce. +/// +/// This message is technically compatible with [`Message`], but exists to +// ensure generated code doesn't use Message. +message NonceMessage { + /// A byte array containing plaintext authenticated data. + bytes aad = 1; + /// An byte array containing the channel ID this message is + /// associated with. A zero-length channel ID is not valid. + bytes channel_id = 2; + /// A potentially encrypted bytestream containing opaque data intended + /// for use in the enclave. + bytes data = 3; + /// The explicit nonce. + fixed64 nonce = 4; +} diff --git a/attest/api/src/conversions.rs b/attest/api/src/conversions.rs index 2467b959c1..e20a825bfe 100644 --- a/attest/api/src/conversions.rs +++ b/attest/api/src/conversions.rs @@ -2,11 +2,11 @@ //! Conversions from gRPC message types into consensus_enclave_api types. -use crate::attest::{AuthMessage, Message}; +use crate::attest::{AuthMessage, Message, NonceMessage}; use mc_attest_ake::{AuthRequestOutput, AuthResponseOutput}; use mc_attest_enclave_api::{ - ClientAuthRequest, ClientAuthResponse, EnclaveMessage, PeerAuthRequest, PeerAuthResponse, - Session, + ClientAuthRequest, ClientAuthResponse, EnclaveMessage, NonceSession, PeerAuthRequest, + PeerAuthResponse, Session, }; use mc_crypto_keys::Kex; use mc_crypto_noise::{HandshakePattern, NoiseCipher, NoiseDigest}; @@ -103,7 +103,31 @@ impl From> for Message { fn from(src: EnclaveMessage) -> Message { let mut retval = Message::default(); retval.set_aad(src.aad); - retval.set_channel_id(src.channel_id.clone().into()); + retval.set_channel_id(src.channel_id.into()); + retval.set_data(src.data); + retval + } +} + +impl From for EnclaveMessage { + fn from(src: NonceMessage) -> Self { + let channel_id = NonceSession::new(src.channel_id, src.nonce); + Self { + aad: src.aad, + channel_id, + data: src.data, + } + } +} + +impl From> for NonceMessage { + fn from(src: EnclaveMessage) -> NonceMessage { + let mut retval = NonceMessage::default(); + retval.set_aad(src.aad); + // it doesn't matter if we don't bump the nonce when retrieving it, + // src.channel_id will be discarded anyways. + retval.set_nonce(src.channel_id.peek_nonce()); + retval.set_channel_id(src.channel_id.into()); retval.set_data(src.data); retval } diff --git a/attest/enclave-api/src/error.rs b/attest/enclave-api/src/error.rs index 8128ffa5fd..0afddb4901 100644 --- a/attest/enclave-api/src/error.rs +++ b/attest/enclave-api/src/error.rs @@ -5,7 +5,7 @@ use core::result::Result as StdResult; use displaydoc::Display; use mc_attest_ake::Error as AkeError; -use mc_attest_core::{NonceError, QuoteError, SgxError}; +use mc_attest_core::{IntelSealingError, NonceError, ParseSealedError, QuoteError, SgxError}; use mc_attest_verifier::Error as VerifierError; use mc_crypto_noise::CipherError; use mc_sgx_compat::sync::PoisonError; @@ -50,6 +50,12 @@ pub enum Error { /// Another thread crashed while holding a lock Poison, + /// An error occurred during a sealing operation + Seal(IntelSealingError), + + /// An error occurred during an unsealing operation + Unseal(ParseSealedError), + /** * Invalid state for call * @@ -109,3 +115,15 @@ impl From for Error { Error::Verify(src) } } + +impl From for Error { + fn from(src: IntelSealingError) -> Error { + Error::Seal(src) + } +} + +impl From for Error { + fn from(src: ParseSealedError) -> Error { + Error::Unseal(src) + } +} diff --git a/attest/enclave-api/src/lib.rs b/attest/enclave-api/src/lib.rs index 848fe9d327..bb1950cae4 100644 --- a/attest/enclave-api/src/lib.rs +++ b/attest/enclave-api/src/lib.rs @@ -12,77 +12,61 @@ mod error; pub use error::{Error, Result}; use alloc::vec::Vec; -use core::hash::Hash; -use mc_attest_core::{QuoteNonce, Report}; +use core::hash::{Hash, Hasher}; +use mc_attest_core::{IntelSealed, QuoteNonce, Report}; use serde::{Deserialize, Serialize}; -/// The raw authentication request message, sent from an initiator to a -/// responder -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct ClientAuthRequest(Vec); +macro_rules! impl_newtype_vec_inout { + ($($newtype:ident;)*) => {$( + impl From> for $newtype { + fn from(src: alloc::vec::Vec) -> $newtype { + $newtype(src) + } + } -impl From> for ClientAuthRequest { - fn from(src: Vec) -> Self { - Self(src) - } + impl From<$newtype> for alloc::vec::Vec { + fn from(src: $newtype) -> alloc::vec::Vec { + src.0 + } + } + )*} } -impl From for Vec { - fn from(src: ClientAuthRequest) -> Vec { - src.0 - } +impl_newtype_vec_inout! { + ClientAuthRequest; ClientAuthResponse; ClientSession; + PeerAuthRequest; PeerAuthResponse; PeerSession; + NonceAuthRequest; NonceAuthResponse; } +/// The raw authentication request message, sent from an initiator to a +/// responder +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct ClientAuthRequest(Vec); + /// The raw authentication response message, sent from a responder to an /// initiator. #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct ClientAuthResponse(Vec); -impl From for Vec { - fn from(src: ClientAuthResponse) -> Vec { - src.0 - } -} - -impl From> for ClientAuthResponse { - fn from(src: Vec) -> Self { - Self(src) - } -} - /// The raw authentication request message, sent from an initiator to a /// responder #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct PeerAuthRequest(Vec); -impl From> for PeerAuthRequest { - fn from(src: Vec) -> Self { - Self(src) - } -} - -impl From for Vec { - fn from(src: PeerAuthRequest) -> Vec { - src.0 - } -} - /// The raw authentication response message, sent from a responder to an /// initiator. #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct PeerAuthResponse(Vec); -impl From for Vec { - fn from(src: PeerAuthResponse) -> Vec { - src.0 - } -} +/// The raw authentication request message, sent from an initiator to a +/// responder. +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct NonceAuthRequest(Vec); -impl From> for PeerAuthResponse { - fn from(src: Vec) -> Self { - Self(src) - } -} +/// The raw authentication response message, sent from a responder to an +/// initiator. +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct NonceAuthResponse(Vec); /// Inbound and outbound messages to/from an enclave. #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] @@ -95,6 +79,27 @@ pub struct EnclaveMessage { pub data: Vec, } +/// Inbound and outbound messages to/from an enclave with an explicit nonce. +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct EnclaveNonceMessage { + /// Authenticated data, if any. + pub aad: Vec, + /// The channel ID of this message. + pub channel_id: S, + /// The encrypted payload data of this message. + pub data: Vec, + /// The explicit nonce for this message. + pub nonce: u64, +} + +/// An EnclaveMessage sealed for the current enclave +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SealedClientMessage { + pub aad: Vec, + pub channel_id: ClientSession, + pub data: IntelSealed, +} + /// The response to a request for a new report. The enclave will expect the /// QuoteNonce to be used when the report is quoted, and both the quote and /// report to be returned to the enclave during the verify_quote() phase. @@ -106,39 +111,35 @@ pub struct NewEReportResponse { /// A helper trait to aid in generic implementation of enclave methods pub trait Session: - Clone + Default + Eq + Hash + for<'bytes> From<&'bytes [u8]> + Into> + Clone + Default + Hash + for<'bytes> From<&'bytes [u8]> + Into> + PartialEq + PartialOrd { type Request: Into>; type Response: From>; } -/// An opaque bytestream used as a client session -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct ClientSession(pub Vec); - -impl AsRef<[u8]> for ClientSession { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } -} +macro_rules! impl_newtype_asref_and_from_bytes { + ($($newtype:ident;)*) => {$( + impl AsRef<[u8]> for $newtype { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } + } -impl<'bytes> From<&'bytes [u8]> for ClientSession { - fn from(src: &[u8]) -> ClientSession { - Self(Vec::from(src)) - } + impl<'bytes> From<&'bytes [u8]> for $newtype { + fn from(src: &[u8]) -> $newtype { + Self(alloc::vec::Vec::from(src)) + } + } + )*} } -impl From> for ClientSession { - fn from(src: Vec) -> ClientSession { - ClientSession(src) - } +impl_newtype_asref_and_from_bytes! { + ClientSession; PeerSession; } -impl From for Vec { - fn from(src: ClientSession) -> Vec { - src.0 - } -} +/// An opaque bytestream used as a client session +#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct ClientSession(Vec); impl Session for ClientSession { type Request = ClientAuthRequest; @@ -149,31 +150,82 @@ impl Session for ClientSession { #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct PeerSession(Vec); -impl AsRef<[u8]> for PeerSession { +impl Session for PeerSession { + type Request = PeerAuthRequest; + type Response = PeerAuthResponse; +} + +/// An opaque bytestream used as a session ID for a session which uses explicit +/// nonces. +#[derive(Clone, Debug, Default, Deserialize, PartialOrd, Serialize)] +pub struct NonceSession { + channel_id: Vec, + nonce: u64, +} + +impl NonceSession { + /// Create a new nonce session from a vector and nonce. + /// + /// This takes a pre-created Vec in order to remove an extra allocation that + /// would be required when converting from a NonceMessage to an + /// [`EnclaveMessage`]`<`[`NonceSession`]`>`. + pub fn new(channel_id: Vec, nonce: u64) -> Self { + Self { channel_id, nonce } + } + + /// Retrieves the nonce for this session + pub fn peek_nonce(&self) -> u64 { + self.nonce + } + + /// Retrieves a copy of the nonce, and increments it for the next time. + pub fn get_nonce(&mut self) -> u64 { + let retval = self.nonce; + self.nonce += 1; + retval + } +} + +impl AsRef<[u8]> for NonceSession { fn as_ref(&self) -> &[u8] { - self.0.as_ref() + self.channel_id.as_ref() } } -impl<'bytes> From<&'bytes [u8]> for PeerSession { - fn from(src: &[u8]) -> PeerSession { - Self(Vec::from(src)) +impl<'bytes> From<&'bytes [u8]> for NonceSession { + fn from(src: &'bytes [u8]) -> Self { + Self::from(Vec::from(src)) } } -impl From> for PeerSession { - fn from(src: Vec) -> PeerSession { - PeerSession(src) +impl From> for NonceSession { + fn from(channel_id: Vec) -> Self { + NonceSession { + channel_id, + nonce: 0, + } } } -impl From for Vec { - fn from(src: PeerSession) -> Vec { - src.0 +impl From for Vec { + fn from(src: NonceSession) -> Self { + src.channel_id } } -impl Session for PeerSession { - type Request = PeerAuthRequest; - type Response = PeerAuthResponse; +impl Hash for NonceSession { + fn hash(&self, state: &mut H) { + self.channel_id.hash(state) + } +} + +impl PartialEq for NonceSession { + fn eq(&self, other: &Self) -> bool { + self.channel_id == other.channel_id + } +} + +impl Session for NonceSession { + type Request = NonceAuthRequest; + type Response = NonceAuthResponse; } diff --git a/crypto/ake/enclave/src/lib.rs b/crypto/ake/enclave/src/lib.rs index 770b77c574..0a06280405 100644 --- a/crypto/ake/enclave/src/lib.rs +++ b/crypto/ake/enclave/src/lib.rs @@ -11,14 +11,14 @@ use mc_attest_ake::{ ClientInitiate, NodeAuthRequestInput, NodeInitiate, Ready, Start, Transition, }; use mc_attest_core::{ - IasNonce, Nonce, NonceError, Quote, QuoteNonce, Report, ReportData, TargetInfo, + IasNonce, IntelSealed, Nonce, NonceError, Quote, QuoteNonce, Report, ReportData, TargetInfo, VerificationReport, }; use mc_attest_enclave_api::{ ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, Error, PeerAuthRequest, - PeerAuthResponse, PeerSession, Result, + PeerAuthResponse, PeerSession, Result, SealedClientMessage, }; -use mc_attest_trusted::EnclaveReport; +use mc_attest_trusted::{EnclaveReport, SealAlgo}; use mc_attest_verifier::{MrEnclaveVerifier, Verifier, DEBUG_ENCLAVE}; use mc_common::{LruCache, ResponderId}; use mc_crypto_keys::{X25519Private, X25519Public, X25519}; @@ -438,24 +438,50 @@ impl AkeEnclaveState { } /// Transforms an incoming client message, i.e. a message sent from a client - /// to the current enclave, into a list of outbound messages for - /// other enclaves that serve as backends to the current enclave. - /// - /// / --> Backend Enclave 1 + /// to the current enclave, into a sealed message which can be decrypted + /// later for use by this enclave without advancing the Noise nonce. + pub fn decrypt_client_message_for_enclave( + &self, + incoming_client_message: EnclaveMessage, + ) -> Result { + let aad = incoming_client_message.aad.clone(); + let channel_id = incoming_client_message.channel_id.clone(); + let client_query_bytes = self.client_decrypt(incoming_client_message)?; + let sealed_data = IntelSealed::seal_raw(&client_query_bytes, &[])?; + + Ok(SealedClientMessage { + channel_id, + aad, + data: sealed_data, + }) + } + + /// Unseals the data component of a sealed client message and returns the + /// plaintext + pub fn unseal(&self, sealed_message: &SealedClientMessage) -> Result> { + Ok(sealed_message.data.unseal_raw()?.0) + } + + /// Transforms a sealed client message, i.e. a message sent from a client + /// to the current enclave which has been sealed for this enclave, into a + /// list of outbound messages for other enclaves that serve as backends to + /// the current enclave. + /// / --> Backend Enclave 1 /// Client -> Current Enclave ---> Backend Enclave 2 /// \ --> Backend Enclave N - pub fn reencrypt_client_message_for_backends( + pub fn reencrypt_sealed_message_for_backends( &self, - incoming_client_message: EnclaveMessage, + sealed_client_message: &SealedClientMessage, ) -> Result>> { - let client_query_bytes: Vec = self.client_decrypt(incoming_client_message.clone())?; + let (client_query_bytes, _) = sealed_client_message.data.unseal_raw()?; + let mut backends = self.backends.lock()?; let backend_messages = backends .iter_mut() .map(|(_, encryptor)| { - let aad = incoming_client_message.aad.clone(); + let aad = sealed_client_message.aad.clone(); let data = encryptor.encrypt(&aad, &client_query_bytes)?; - let channel_id = incoming_client_message.channel_id.clone(); + let channel_id = sealed_client_message.channel_id.clone(); Ok(EnclaveMessage { aad, channel_id, @@ -467,6 +493,19 @@ impl AkeEnclaveState { Ok(backend_messages) } + pub fn backend_decrypt( + &self, + responder_id: ResponderId, + msg: EnclaveMessage, + ) -> Result> { + // Ensure lock gets released as soon as we're done decrypting. + let mut backends = self.backends.lock()?; + backends + .get_mut(&responder_id) + .ok_or(Error::NotFound) + .and_then(|session| Ok(session.decrypt(&msg.aad, &msg.data)?)) + } + // // IAS related // diff --git a/fog/api/proto/ledger.proto b/fog/api/proto/ledger.proto index 908be70ade..ff0edbf6db 100644 --- a/fog/api/proto/ledger.proto +++ b/fog/api/proto/ledger.proto @@ -60,15 +60,33 @@ message MultiKeyImageStoreRequest { repeated attest.Message queries = 1; } + +/// The status associated with a MultiViewStoreQueryResponse +enum MultiKeyImageStoreResponseStatus { + /// The Fog Ledger Store successfully fulfilled the request. + SUCCESS = 0; + /// The Fog Ledger Store is unable to decrypt a query within the MultiKeyImageStoreRequest. It needs to be authenticated + /// by the router. + AUTHENTICATION_ERROR = 1; + /// The Fog Ledger Store is not ready to service a MultiViewStoreQueryRequest. This might be because the store has + /// not loaded enough blocks yet. + NOT_READY = 2; +} + message MultiKeyImageStoreResponse { /// Optional field that gets set when the Fog Ledger Store is able to decrypt a query /// included in the MultiKeyImageStoreRequest and create a query response for that // query. attest.Message query_response = 1; - /// Optional error that gets returned when the Fog Ledger Store - /// cannot decrypt the MultiKeyImageStoreRequest. - FogLedgerStoreDecryptionError decryption_error = 2; + /// The FogViewStoreUri for the specific Fog View Store that + /// tried to decrypt the MultiViewStoreQueryRequest and failed. + /// The client should subsequently authenticate with the machine + /// described by this URI. + string fog_ledger_store_uri = 2; + + /// Status that gets returned when the Fog Ledger Store services a MultiKeyImageStoreRequest. + MultiKeyImageStoreResponseStatus status = 3; } //// diff --git a/fog/api/proto/view.proto b/fog/api/proto/view.proto index e18799349e..7b7c8a2b51 100644 --- a/fog/api/proto/view.proto +++ b/fog/api/proto/view.proto @@ -10,12 +10,24 @@ import "external.proto"; import "kex_rng.proto"; import "fog_common.proto"; +import "google/protobuf/empty.proto"; + /// A single Duplex streaming API that allows clients to authorize with Fog View and /// query it for TxOuts. service FogViewRouterAPI { rpc request(stream FogViewRouterRequest) returns (stream FogViewRouterResponse) {} } +service FogViewRouterAdminAPI { + // Adds a shard to the Fog View Router's list of shards to query. + rpc addShard(AddShardRequest) returns (google.protobuf.Empty) {} +} + +message AddShardRequest { + // The shard's URI in string format. + string shard_uri = 1; +} + message FogViewRouterRequest { oneof request_data { /// This is called to perform IX key exchange @@ -36,31 +48,37 @@ message FogViewRouterResponse { } } -message FogViewStoreDecryptionError { - /// The FogViewStoreUri for the specific Fog View Store that - /// tried to decrypt the MultiViewStoreQueryRequest and failed. - /// The client should subsequently authenticate with the machine - /// described by this URI. - string fog_view_store_uri = 1; - - /// An error message that describes the decryption error. - string error_message = 2; -} - message MultiViewStoreQueryRequest { /// A list of queries encrypted for Fog View Stores. repeated attest.Message queries = 1; } +/// The status associated with a MultiViewStoreQueryResponse +enum MultiViewStoreQueryResponseStatus { + /// The Fog View Store successfully fulfilled the request. + SUCCESS = 0; + /// The Fog View Store is unable to decrypt a query within the MultiViewStoreQuery. It needs to be authenticated + /// by the router. + AUTHENTICATION_ERROR = 1; + /// The Fog View Store is not ready to service a MultiViewStoreQueryRequest. This might be because the store has + /// not loaded enough blocks yet. + NOT_READY = 2; +} + message MultiViewStoreQueryResponse { /// Optional field that gets set when the Fog View Store is able to decrypt a query /// included in the MultiViewStoreQueryRequest and create a query response for that // query. attest.Message query_response = 1; - /// Optional error that gets returned when the Fog View Store - /// cannot decrypt the MultiViewStoreQuery. - FogViewStoreDecryptionError decryption_error = 2; + /// The FogViewStoreUri for the specific Fog View Store that + /// tried to decrypt the MultiViewStoreQueryRequest and failed. + /// The client should subsequently authenticate with the machine + /// described by this URI. + string fog_view_store_uri = 2; + + /// Status that gets returned when the Fog View Store services a MultiViewStoreQueryRequest. + MultiViewStoreQueryResponseStatus status = 3; } /// Fulfills requests sent directly by a Fog client, e.g. a mobile phone using the SDK. diff --git a/fog/api/src/conversions.rs b/fog/api/src/conversions.rs index 965ccb88c3..31db80fcbf 100644 --- a/fog/api/src/conversions.rs +++ b/fog/api/src/conversions.rs @@ -2,7 +2,9 @@ // // Contains helper methods that enable conversions for Fog Api types. -use crate::{fog_common, ingest_common, view::MultiViewStoreQueryRequest}; +use crate::{ + fog_common, ingest_common, ledger::MultiKeyImageStoreRequest, view::MultiViewStoreQueryRequest, +}; use mc_api::ConversionError; use mc_attest_api::attest; use mc_attest_enclave_api::{ClientSession, EnclaveMessage}; @@ -21,6 +23,18 @@ impl From>> + for MultiKeyImageStoreRequest +{ + fn from(enclave_messages: Vec>) -> MultiKeyImageStoreRequest { + enclave_messages + .into_iter() + .map(|enclave_message| enclave_message.into()) + .collect::>() + .into() + } +} + impl From> for MultiViewStoreQueryRequest { fn from(attested_query_messages: Vec) -> MultiViewStoreQueryRequest { let mut multi_view_store_query_request = MultiViewStoreQueryRequest::new(); @@ -30,6 +44,15 @@ impl From> for MultiViewStoreQueryRequest { } } +impl From> for MultiKeyImageStoreRequest { + fn from(attested_query_messages: Vec) -> MultiKeyImageStoreRequest { + let mut multi_key_image_store_request = MultiKeyImageStoreRequest::new(); + multi_key_image_store_request.set_queries(attested_query_messages.into()); + + multi_key_image_store_request + } +} + impl From<&common::BlockRange> for fog_common::BlockRange { fn from(common_block_range: &common::BlockRange) -> fog_common::BlockRange { let mut proto_block_range = fog_common::BlockRange::new(); diff --git a/fog/ledger/connection/src/error.rs b/fog/ledger/connection/src/error.rs index 3b2e11d2df..2a0cb9238a 100644 --- a/fog/ledger/connection/src/error.rs +++ b/fog/ledger/connection/src/error.rs @@ -8,6 +8,7 @@ use mc_api::ConversionError; use mc_fog_enclave_connection::Error as EnclaveConnectionError; use mc_fog_uri::FogLedgerUri; +use mc_util_uri::UriConversionError; /// Error type returned by LedgerServerConn #[derive(Debug, Display)] @@ -35,3 +36,9 @@ impl From for Error { Error::Conversion(err) } } + +impl From for Error { + fn from(err: UriConversionError) -> Self { + Self::UriConversionError(err) + } +} diff --git a/fog/ledger/enclave/api/src/lib.rs b/fog/ledger/enclave/api/src/lib.rs index edf2115134..c35fb48d41 100644 --- a/fog/ledger/enclave/api/src/lib.rs +++ b/fog/ledger/enclave/api/src/lib.rs @@ -13,9 +13,11 @@ pub use crate::{ error::{AddRecordsError, Error}, messages::{EnclaveCall, KeyImageData}, }; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; use core::result::Result as StdResult; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; use mc_common::ResponderId; use mc_crypto_keys::X25519Public; pub use mc_fog_types::ledger::{ @@ -99,12 +101,13 @@ pub trait LedgerEnclave: ReportableEnclave { /// Add a key image data to the oram Using thrm -rf targete key image fn add_key_image_data(&self, records: Vec) -> Result<()>; - // LEDGER ROUTER / STORE SYSTEM - /// Begin a connection to a Fog Ledger Store. The enclave calling this method, - /// most likely a router, will act as a client to the Fog Ledger Store. - fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) -> Result; + /// Begin a connection to a Fog Ledger Store. The enclave calling this + /// method, most likely a router, will act as a client to the Fog Ledger + /// Store. + fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) + -> Result; /// Complete the connection to a Fog Ledger Store that has accepted our /// ClientAuthRequest. This is meant to be called after the enclave has @@ -114,22 +117,38 @@ pub trait LedgerEnclave: ReportableEnclave { ledger_store_id: ResponderId, ledger_store_auth_response: ClientAuthResponse, ) -> Result<()>; - + + /// Decrypts a client query message and converts it into a + /// SealedClientMessage which can be unsealed multiple times to + /// construct the MultiKeyImageStoreRequest. + fn decrypt_and_seal_query( + &self, + client_query: EnclaveMessage, + ) -> Result; + /// Transforms a client query request into a list of query request data. /// /// The returned list is meant to be used to construct the - /// MultiLedgerStoreQuery, which is sent to each shard. - fn create_key_image_store_query( + /// MultiKeyImageStoreRequest, which is sent to each shard. + fn create_multi_key_image_store_query_data( &self, - client_query: EnclaveMessage, + sealed_query: SealedClientMessage, ) -> Result>>; - /// Used by a Ledger Store to handle an inbound encrypted ledger.proto LedgerRequest. - /// Generally, these come in from a router. - /// This could could be a key image request, a merkele proof + /// Receives all of the shards' query responses and collates them into one + /// query response for the client. + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result>; + + /// Used by a Ledger Store to handle an inbound encrypted ledger.proto + /// LedgerRequest. Generally, these come in from a router. + /// This could could be a key image request, a merkele proof /// request, and potentially in the future an untrusted tx out request. fn handle_key_image_store_request( - &self, + &self, router_query: EnclaveMessage, ) -> Result>; } diff --git a/fog/ledger/enclave/api/src/messages.rs b/fog/ledger/enclave/api/src/messages.rs index 332727a7c5..f4fc4cc0ac 100644 --- a/fog/ledger/enclave/api/src/messages.rs +++ b/fog/ledger/enclave/api/src/messages.rs @@ -2,9 +2,13 @@ //! The message types used by the ledger_enclave_api. use crate::UntrustedKeyImageQueryResponse; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; use mc_attest_core::{Quote, Report, TargetInfo, VerificationReport}; -use mc_attest_enclave_api::{ClientAuthRequest, ClientSession, EnclaveMessage, ClientAuthResponse}; + +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; + use mc_common::ResponderId; use mc_fog_types::ledger::GetOutputsResponse; use mc_transaction_core::ring_signature::KeyImage; @@ -34,7 +38,7 @@ pub struct KeyImageData { /// An enumeration of API calls and their arguments for use across serialization /// boundaries. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum EnclaveCall { /// The [LedgerEnclave::enclave_init()] method. EnclaveInit(ResponderId, u64), @@ -104,8 +108,10 @@ pub enum EnclaveCall { /// The [LedgerEnclave::connect_to_store()] method. /// - /// Begin a connection to a Fog Ledger Store. The enclave calling this method, - /// most likely a router, will act as a client to the Fog Ledger Store. + /// Begin a connection to a Fog Ledger Store. The enclave calling this + /// method, most likely a router, will act as a client to the Fog Ledger + /// Store. + ConnectToKeyImageStore(ResponderId), /// The [LedgerEnclave::finish_connecting_to_store()] method. @@ -113,24 +119,34 @@ pub enum EnclaveCall { /// Complete the connection to a Fog Ledger Store that has accepted our /// ClientAuthRequest. This is meant to be called after the enclave has /// initialized and discovers a new Fog Ledger Store. - FinishConnectingToKeyImageStore( - ResponderId, - ClientAuthResponse, - ), - - /// The [LedgerEnclave::create_key_image_store_query()] method. + FinishConnectingToKeyImageStore(ResponderId, ClientAuthResponse), + + /// The [LedgerEnclave::decrypt_and_seal_query()] method. + /// + /// Takes a client query message and returns a SealedClientMessage + /// sealed for the current enclave. + DecryptAndSealQuery(EnclaveMessage), + + /// The [LedgerEnclave::create_multi_key_image_store_query()] method. /// /// Transforms a client query request into a list of query request data. /// /// The returned list is meant to be used to construct the /// MultiKeyImageStoreRequest, which is sent to each shard. - CreateKeyImageStoreQuery(EnclaveMessage), + CreateMultiKeyImageStoreQueryData(SealedClientMessage), + + /// Collates shard query responses into a single query response for the + /// client. + CollateQueryResponses( + SealedClientMessage, + BTreeMap>, + ), /// The [LedgerEnclave::handle_key_image_store_request()] method. /// - /// Used by a Ledger Store to handle an inbound encrypted ledger.proto LedgerRequest. - /// Generally, these come in from a router. - /// This could could be a key image request, a merkele proof + /// Used by a Ledger Store to handle an inbound encrypted ledger.proto + /// LedgerRequest. Generally, these come in from a router. + /// This could could be a key image request, a merkele proof /// request, and potentially in the future an untrusted tx out request. HandleKeyImageStoreRequest(EnclaveMessage), } \ No newline at end of file diff --git a/fog/ledger/enclave/impl/Cargo.toml b/fog/ledger/enclave/impl/Cargo.toml index b0d9597e66..17326853d7 100644 --- a/fog/ledger/enclave/impl/Cargo.toml +++ b/fog/ledger/enclave/impl/Cargo.toml @@ -12,6 +12,7 @@ license = "GPL-3.0" # mobilecoin mc-attest-core = { path = "../../../../attest/core", default-features = false } mc-attest-enclave-api = { path = "../../../../attest/enclave-api", default-features = false } +mc-blockchain-types = { path = "../../../../blockchain/types" } mc-common = { path = "../../../../common", default-features = false } mc-crypto-ake-enclave = { path = "../../../../crypto/ake/enclave", default-features = false } mc-crypto-keys = { path = "../../../../crypto/keys", default-features = false } diff --git a/fog/ledger/enclave/impl/src/lib.rs b/fog/ledger/enclave/impl/src/lib.rs index 36b73a0a19..325a4d472f 100644 --- a/fog/ledger/enclave/impl/src/lib.rs +++ b/fog/ledger/enclave/impl/src/lib.rs @@ -13,10 +13,14 @@ extern crate alloc; mod key_image_store; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; +use core::cmp::max; use key_image_store::{KeyImageStore, StorageDataSize, StorageMetaSize}; use mc_attest_core::{IasNonce, Quote, QuoteNonce, Report, TargetInfo, VerificationReport}; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; +use mc_blockchain_types::MAX_BLOCK_VERSION; use mc_common::{ logger::{log, Logger}, ResponderId, @@ -192,8 +196,14 @@ where Ok(()) } - fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) -> Result { - mc_sgx_debug::eprintln!("Called connect_to_key_image_store(ledger_store_id: {})", ledger_store_id); + fn connect_to_key_image_store( + &self, + ledger_store_id: ResponderId, + ) -> Result { + mc_sgx_debug::eprintln!( + "Called connect_to_key_image_store(ledger_store_id: {})", + ledger_store_id + ); Ok(self.ake.backend_init(ledger_store_id)?) } @@ -209,20 +219,105 @@ where .backend_connect(ledger_store_id, ledger_store_auth_response)?) } - fn create_key_image_store_query( + fn decrypt_and_seal_query( &self, client_query: EnclaveMessage, + ) -> Result { + Ok(self.ake.decrypt_client_message_for_enclave(client_query)?) + } + + fn create_multi_key_image_store_query_data( + &self, + sealed_query: SealedClientMessage, ) -> Result>> { - mc_sgx_debug::eprintln!("Called create_key_image_store_query(..)"); + mc_sgx_debug::eprintln!("Called create_multi_key_image_store_query_data(..)"); Ok(self .ake - .reencrypt_client_message_for_backends(client_query)?) + .reencrypt_sealed_message_for_backends(&sealed_query)?) } + + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result> { + if shard_query_responses.is_empty() { + return Ok(EnclaveMessage::default()); + } + let channel_id = sealed_query.channel_id.clone(); + let client_query_plaintext = self.ake.unseal(&sealed_query)?; + // TODO this will (possibly?) be used when we implement obliviousness + let _client_query_request: CheckKeyImagesRequest = + mc_util_serial::decode(&client_query_plaintext).map_err(|e| { + log::error!(self.logger, "Could not decode client query request: {}", e); + Error::ProstDecode + })?; + + let shard_query_responses = shard_query_responses + .into_iter() + .map(|(responder_id, enclave_message)| { + let plaintext_bytes = self.ake.backend_decrypt(responder_id, enclave_message)?; // TODO explicit nonces + let query_response: CheckKeyImagesResponse = + mc_util_serial::decode(&plaintext_bytes)?; + + Ok(query_response) + }) + .collect::>>()?; + + // NOTES: + // num_blocks = min(responses.num_blocks) + // global_txo_count = min(global_txo_count) TODO CONFIRM + // results = cat(responses.results) + // latest_block_version = max(responses.latest_block_version) + // max_block_version = max(latest_block_version, + // mc_transaction_core::MAX_BLOCK_VERSION + + // TODO no unwraps + let num_blocks = shard_query_responses + .iter() + .map(|query_response| query_response.num_blocks) + .min() + .unwrap(); + let global_txo_count = shard_query_responses + .iter() + .map(|query_response| query_response.global_txo_count) + .min() + .unwrap(); + let latest_block_version = shard_query_responses + .iter() + .map(|query_response| query_response.latest_block_version) + .max() + .unwrap(); + // TODO I believe this needs to be implemented in an oblivious way to meet the + // security requirements. I'm not 100% sure what an oblivious approach + // to this looks like, though. In general this kind of thing needs to be + // talked about. + let results = shard_query_responses + .into_iter() + .flat_map(|query_response| query_response.results) + .collect(); + let max_block_version = max(latest_block_version, *MAX_BLOCK_VERSION); + + let client_query_response = CheckKeyImagesResponse { + num_blocks, + global_txo_count, + results, + latest_block_version, + max_block_version, + }; + let response_plaintext_bytes = mc_util_serial::encode(&client_query_response); + let response = + self.ake + .client_encrypt(&channel_id, &sealed_query.aad, &response_plaintext_bytes)?; + + Ok(response) + } + fn handle_key_image_store_request( - &self, - _: EnclaveMessage - ) -> Result> { - todo!() + &self, + _: EnclaveMessage, + ) -> Result> { + todo!() } } diff --git a/fog/ledger/enclave/src/lib.rs b/fog/ledger/enclave/src/lib.rs index e29d2170ee..ed3ccdefb2 100644 --- a/fog/ledger/enclave/src/lib.rs +++ b/fog/ledger/enclave/src/lib.rs @@ -14,7 +14,9 @@ pub use mc_fog_ledger_enclave_api::{ use mc_attest_core::{ IasNonce, Quote, QuoteNonce, Report, SgxError, TargetInfo, VerificationReport, }; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; use mc_attest_verifier::DEBUG_ENCLAVE; use mc_common::{logger::Logger, ResponderId}; use mc_crypto_keys::X25519Public; @@ -25,7 +27,7 @@ use mc_sgx_types::{ sgx_attributes_t, sgx_enclave_id_t, sgx_launch_token_t, sgx_misc_attribute_t, sgx_status_t, }; use mc_sgx_urts::SgxEnclave; -use std::{path, result::Result as StdResult, sync::Arc}; +use std::{collections::BTreeMap, path, result::Result as StdResult, sync::Arc}; /// The default filename of the fog ledger's SGX enclave binary. pub const ENCLAVE_FILE: &str = "libledger-enclave.signed.so"; @@ -189,9 +191,16 @@ impl LedgerEnclave for LedgerSgxEnclave { } // Router/store system. - fn connect_to_key_image_store(&self, ledger_store_id: ResponderId) -> Result { - mc_sgx_debug::eprintln!("Called connect_to_key_image_store(ledger_store_id: {})", ledger_store_id); - let inbuf = mc_util_serial::serialize(&EnclaveCall::ConnectToKeyImageStore(ledger_store_id))?; + fn connect_to_key_image_store( + &self, + ledger_store_id: ResponderId, + ) -> Result { + mc_sgx_debug::eprintln!( + "Called connect_to_key_image_store(ledger_store_id: {})", + ledger_store_id + ); + let inbuf = + mc_util_serial::serialize(&EnclaveCall::ConnectToKeyImageStore(ledger_store_id))?; let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } @@ -203,17 +212,49 @@ impl LedgerEnclave for LedgerSgxEnclave { ledger_store_auth_response: ClientAuthResponse, ) -> Result<()> { mc_sgx_debug::eprintln!("Called finish_connecting_to_key_image_store(ledger_store_id: {}, ledger_store_auth_response: {:?})", ledger_store_id, ledger_store_auth_response); - let inbuf = mc_util_serial::serialize(&EnclaveCall::FinishConnectingToKeyImageStore(ledger_store_id, ledger_store_auth_response))?; + + let inbuf = mc_util_serial::serialize(&EnclaveCall::FinishConnectingToKeyImageStore( + ledger_store_id, + ledger_store_auth_response, + ))?; + let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } - fn create_key_image_store_query( + fn decrypt_and_seal_query( &self, client_query: EnclaveMessage, + ) -> Result { + mc_sgx_debug::eprintln!( + "Called decrypt_and_seal_query(..) - the router is handling a message from the client" + ); + let inbuf = mc_util_serial::serialize(&EnclaveCall::DecryptAndSealQuery(client_query))?; + let outbuf = self.enclave_call(&inbuf)?; + mc_util_serial::deserialize(&outbuf[..])? + } + + fn create_multi_key_image_store_query_data( + &self, + sealed_query: SealedClientMessage, ) -> Result>> { - mc_sgx_debug::eprintln!("Called create_key_image_store_query(..) - the router is handling a message from the client"); - let inbuf = mc_util_serial::serialize(&EnclaveCall::CreateKeyImageStoreQuery(client_query))?; + mc_sgx_debug::eprintln!("Called create_multi_key_image_store_query_data(..) - the router is handling a message from the client"); + let inbuf = mc_util_serial::serialize(&EnclaveCall::CreateMultiKeyImageStoreQueryData( + sealed_query, + ))?; + let outbuf = self.enclave_call(&inbuf)?; + mc_util_serial::deserialize(&outbuf[..])? + } + + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result> { + let inbuf = mc_util_serial::serialize(&EnclaveCall::CollateQueryResponses( + sealed_query, + shard_query_responses, + ))?; let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } @@ -223,7 +264,10 @@ impl LedgerEnclave for LedgerSgxEnclave { router_query: EnclaveMessage, ) -> Result> { mc_sgx_debug::eprintln!("Called handle_key_image_store_request(..) - the store is handling a message from the router."); - let inbuf = mc_util_serial::serialize(&EnclaveCall::HandleKeyImageStoreRequest(router_query))?; + + let inbuf = + mc_util_serial::serialize(&EnclaveCall::HandleKeyImageStoreRequest(router_query))?; + let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? } diff --git a/fog/ledger/enclave/trusted/Cargo.lock b/fog/ledger/enclave/trusted/Cargo.lock index ac316e2279..477ae5d758 100644 --- a/fog/ledger/enclave/trusted/Cargo.lock +++ b/fog/ledger/enclave/trusted/Cargo.lock @@ -795,6 +795,29 @@ dependencies = [ "serde", ] +[[package]] +name = "mc-blockchain-types" +version = "1.3.0-pre0" +dependencies = [ + "displaydoc", + "hex_fmt", + "mc-account-keys", + "mc-attest-verifier-types", + "mc-common", + "mc-consensus-scp-types", + "mc-crypto-digestible", + "mc-crypto-digestible-signature", + "mc-crypto-keys", + "mc-crypto-ring-signature", + "mc-transaction-core", + "mc-transaction-types", + "mc-util-from-random", + "mc-util-repr-bytes", + "prost", + "serde", + "zeroize", +] + [[package]] name = "mc-common" version = "1.3.0-pre0" @@ -816,6 +839,18 @@ dependencies = [ "slog", ] +[[package]] +name = "mc-consensus-scp-types" +version = "1.3.0-pre0" +dependencies = [ + "mc-common", + "mc-crypto-digestible", + "mc-crypto-keys", + "mc-util-from-random", + "prost", + "serde", +] + [[package]] name = "mc-crypto-ake-enclave" version = "1.3.0-pre0" @@ -1059,6 +1094,7 @@ dependencies = [ "aligned-cmov", "mc-attest-core", "mc-attest-enclave-api", + "mc-blockchain-types", "mc-common", "mc-crypto-ake-enclave", "mc-crypto-keys", diff --git a/fog/ledger/enclave/trusted/src/lib.rs b/fog/ledger/enclave/trusted/src/lib.rs index 79c6ad38ef..15224b4155 100644 --- a/fog/ledger/enclave/trusted/src/lib.rs +++ b/fog/ledger/enclave/trusted/src/lib.rs @@ -64,16 +64,27 @@ pub fn ecall_dispatcher(inbuf: &[u8]) -> Result, sgx_status_t> { // Router / Store system // Router-side - EnclaveCall::ConnectToKeyImageStore(responder_id) => serialize(&ENCLAVE.connect_to_key_image_store(responder_id)), - EnclaveCall::FinishConnectingToKeyImageStore( - responder_id, - client_auth_response, - ) => serialize(&ENCLAVE.finish_connecting_to_key_image_store(responder_id, client_auth_response)), - EnclaveCall::CreateKeyImageStoreQuery(msg) => - serialize(&ENCLAVE.create_key_image_store_query(msg)), + EnclaveCall::ConnectToKeyImageStore(responder_id) => { + serialize(&ENCLAVE.connect_to_key_image_store(responder_id)) + } + EnclaveCall::FinishConnectingToKeyImageStore(responder_id, client_auth_response) => { + serialize( + &ENCLAVE.finish_connecting_to_key_image_store(responder_id, client_auth_response), + ) + } + EnclaveCall::DecryptAndSealQuery(client_query) => { + serialize(&ENCLAVE.decrypt_and_seal_query(client_query)) + } + EnclaveCall::CreateMultiKeyImageStoreQueryData(msg) => { + serialize(&ENCLAVE.create_multi_key_image_store_query_data(msg)) + } + EnclaveCall::CollateQueryResponses(sealed_query, shard_query_responses) => { + serialize(&ENCLAVE.collate_shard_query_responses(sealed_query, shard_query_responses)) + } // Store-side - EnclaveCall::HandleKeyImageStoreRequest(msg) => - serialize(&ENCLAVE.handle_key_image_store_request(msg)), + EnclaveCall::HandleKeyImageStoreRequest(msg) => { + serialize(&ENCLAVE.handle_key_image_store_request(msg)) + } } .or(Err(sgx_status_t::SGX_ERROR_UNEXPECTED)) } diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 0a4b270c13..6ede472a16 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -15,7 +15,7 @@ path = "src/bin/main.rs" [[bin]] name = "ledger_router_server" -path = "src/bin/router.rs" +path = "src/bin/key_image_router.rs" [dependencies] mc-attest-api = { path = "../../../attest/api" } diff --git a/fog/ledger/server/src/bin/key_image_router.rs b/fog/ledger/server/src/bin/key_image_router.rs index 9106ae33f6..a7424dad98 100644 --- a/fog/ledger/server/src/bin/key_image_router.rs +++ b/fog/ledger/server/src/bin/key_image_router.rs @@ -1,13 +1,15 @@ -use std::{env, sync::Arc, str::FromStr}; +// Copyright (c) 2018-2022 The MobileCoin Foundation +use std::{env, str::FromStr, sync::Arc}; + +use clap::Parser; use grpcio::ChannelBuilder; use mc_common::logger::log; use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; -use mc_fog_ledger_enclave::{ENCLAVE_FILE, LedgerSgxEnclave}; -use mc_fog_ledger_server::LedgerRouterConfig; -use mc_fog_ledger_server::KeyImageRouterServer; -use clap::Parser; -use mc_fog_uri::{KeyImageStoreUri, KeyImageStoreScheme}; +use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; +use mc_fog_ledger_server::{KeyImageRouterServer, LedgerRouterConfig}; +use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; + use mc_util_grpc::ConnectionUriGrpcioChannel; use mc_util_uri::UriScheme; @@ -32,7 +34,7 @@ fn main() { config.omap_capacity, logger.clone(), ); - + let mut ledger_store_grpc_clients: Vec = Vec::new(); let grpc_env = Arc::new( grpcio::EnvBuilder::new() @@ -60,4 +62,4 @@ fn main() { loop { std::thread::sleep(std::time::Duration::from_millis(1000)); } -} \ No newline at end of file +} diff --git a/fog/ledger/server/src/config.rs b/fog/ledger/server/src/config.rs index eb2e2cb43c..d3f0dfc369 100644 --- a/fog/ledger/server/src/config.rs +++ b/fog/ledger/server/src/config.rs @@ -11,7 +11,7 @@ use mc_fog_uri::{FogLedgerUri, KeyImageRouterUri, KeyImageStoreUri}; use mc_util_parse::parse_duration_in_seconds; use mc_util_uri::AdminUri; use serde::Serialize; -use std::{path::PathBuf, time::Duration, str::FromStr}; +use std::{path::PathBuf, str::FromStr, time::Duration}; /// Configuration parameters for the ledger server #[derive(Clone, Parser, Serialize)] @@ -75,8 +75,7 @@ pub struct LedgerServerConfig { /// A Fog Server can either fulfill client requests directly or fulfill Fog /// Ledger Router requests, and these types of servers use different URLs. -/// -/// TODO - This is almost identical to Fog View's implementation of this +/// TODO - This is almost identical to Fog View's implementation of this /// combine it later? #[derive(Clone, Serialize)] pub enum KeyImageClientListenUri { @@ -96,7 +95,10 @@ impl FromStr for KeyImageClientListenUri { return Ok(KeyImageClientListenUri::Store(ledger_store_uri)); } - Err(format!("Incorrect KeyImageClientListenUri string: {}.", input)) + Err(format!( + "Incorrect KeyImageClientListenUri string: {}.", + input + )) } } diff --git a/fog/ledger/server/src/error.rs b/fog/ledger/server/src/error.rs index dbeeefb014..521e49879a 100644 --- a/fog/ledger/server/src/error.rs +++ b/fog/ledger/server/src/error.rs @@ -1,3 +1,5 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + use displaydoc::Display; use grpcio::RpcStatus; use mc_common::logger::Logger; @@ -5,7 +7,6 @@ use mc_fog_ledger_enclave_api::Error as LedgerEnclaveError; use mc_sgx_report_cache_untrusted::Error as ReportCacheError; use mc_util_grpc::{rpc_internal_error, rpc_permissions_error}; - #[derive(Debug, Display)] pub enum RouterServerError { /// Error related to contacting Fog Ledger Store: {0} @@ -32,7 +33,12 @@ impl From for RouterServerError { } } -#[allow(dead_code)] //FIXME +impl From for RouterServerError { + fn from(src: mc_util_uri::UriConversionError) -> Self { + RouterServerError::LedgerStoreError(format!("{}", src)) + } +} + pub fn router_server_err_to_rpc_status( context: &str, src: RouterServerError, @@ -54,7 +60,7 @@ impl From for RouterServerError { } } -#[allow(dead_code)] // FIXME when the ledger router is more than just a skeleton. +#[allow(dead_code)] // FIXME when the ledger router is more than just a skeleton. #[derive(Debug, Display)] pub enum LedgerServerError { /// Ledger Enclave error: {0} diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs index 608bd343c9..1eb71d8c4a 100644 --- a/fog/ledger/server/src/key_image_router_server.rs +++ b/fog/ledger/server/src/key_image_router_server.rs @@ -7,9 +7,9 @@ use mc_common::logger::{log, Logger}; use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::ConnectionUri; -use mc_util_grpc::{ReadinessIndicator, ConnectionUriGrpcioServer}; +use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; -use crate::{key_image_router_service::KeyImageRouterService, config::LedgerRouterConfig}; +use crate::{config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService}; #[allow(dead_code)] // FIXME pub struct KeyImageRouterServer { @@ -20,7 +20,7 @@ pub struct KeyImageRouterServer { impl KeyImageRouterServer { /// Creates a new ledger router server instance #[allow(dead_code)] // FIXME - pub fn new ( + pub fn new( config: LedgerRouterConfig, enclave: E, shards: Vec, @@ -39,14 +39,12 @@ impl KeyImageRouterServer { // Health check service - will be used in both cases let health_service = - mc_util_grpc::HealthService::new( - Some(readiness_indicator.into()), logger.clone() - ).into_service(); + mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) + .into_service(); match config.client_listen_uri { // Router server crate::config::KeyImageClientListenUri::ClientFacing(ledger_router_uri) => { - // Init ledger router service. let ledger_router_service = ledger_grpc::create_ledger_api( KeyImageRouterService::new(enclave, shards, logger.clone()), @@ -63,15 +61,14 @@ impl KeyImageRouterServer { .register_service(ledger_router_service) .register_service(health_service) .bind_using_uri(&ledger_router_uri, logger.clone()); - let server = server_builder.build().unwrap(); - + Self { server, logger } - }, - // Store server. + } + // Store server. crate::config::KeyImageClientListenUri::Store(_ledger_store_uri) => { todo!() - }, + } } } diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs index 3da6a18790..2a2e3be33a 100644 --- a/fog/ledger/server/src/key_image_router_service.rs +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -1,11 +1,13 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + use std::sync::Arc; -use futures::{TryFutureExt, FutureExt}; +use futures::{FutureExt, TryFutureExt}; use grpcio::{DuplexSink, RequestStream, RpcContext}; use mc_common::logger::{log, Logger}; use mc_fog_api::{ - ledger_grpc::{LedgerApi, self}, - ledger::{LedgerRequest, LedgerResponse}, + ledger::{LedgerRequest, LedgerResponse}, + ledger_grpc::{self, LedgerApi}, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_util_grpc::rpc_logger; @@ -28,7 +30,11 @@ impl KeyImageRouterService { /// Creates a new LedgerRouterService that can be used by a gRPC server to /// fulfill gRPC requests. #[allow(dead_code)] // FIXME - pub fn new(enclave: E, shards: Vec, logger: Logger) -> Self { + pub fn new( + enclave: E, + shards: Vec, + logger: Logger, + ) -> Self { let shards = shards.into_iter().map(Arc::new).collect(); Self { enclave, @@ -52,8 +58,10 @@ where log::info!(self.logger, "Request received in request fn"); let _timer = SVC_COUNTERS.req(&ctx); mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { - log::warn!(self.logger, "Streaming GRPC Ledger API only partially implemented."); - + log::warn!( + self.logger, + "Streaming GRPC Ledger API only partially implemented." + ); let logger = logger.clone(); let future = router_handlers::handle_requests( @@ -64,10 +72,10 @@ where logger.clone(), ) .map_err(move |err: grpcio::Error| log::error!(&logger, "failed to reply: {}", err)) - // TODO: Do more with the error than just push it to the log. + // TODO: Do more with the error than just push it to the log. .map(|_| ()); ctx.spawn(future) }); } -} \ No newline at end of file +} diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index c14867f0d1..d038017624 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -7,9 +7,9 @@ use mc_attest_api::{ }; use mc_blockchain_types::MAX_BLOCK_VERSION; use mc_common::logger::{log, Logger}; -use mc_fog_api::{ - ledger_grpc::{FogKeyImageApi, KeyImageStoreApi}, +use mc_fog_api::{ ledger::{MultiKeyImageStoreRequest, MultiKeyImageStoreResponse}, + ledger_grpc::{FogKeyImageApi, KeyImageStoreApi}, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_ledger_enclave_api::{Error as EnclaveError, UntrustedKeyImageQueryResponse}; @@ -172,12 +172,22 @@ impl FogKeyImageApi for KeyImageServic impl KeyImageStoreApi for KeyImageService { #[allow(unused_variables)] //FIXME - fn auth(&mut self, ctx: grpcio::RpcContext, req: AuthMessage, sink: grpcio::UnarySink) { + fn auth( + &mut self, + ctx: grpcio::RpcContext, + req: AuthMessage, + sink: grpcio::UnarySink, + ) { todo!() } #[allow(unused_variables)] //FIXME - fn multi_key_image_store_query(&mut self, ctx: grpcio::RpcContext, req: MultiKeyImageStoreRequest, sink: grpcio::UnarySink) { + fn multi_key_image_store_query( + &mut self, + ctx: grpcio::RpcContext, + req: MultiKeyImageStoreRequest, + sink: grpcio::UnarySink, + ) { todo!() } -} \ No newline at end of file +} diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index 70694bb27a..0dfa1982e0 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -11,9 +11,10 @@ mod router_handlers; mod server; mod untrusted_tx_out_service; -//Router & store system. KeyImageService can function as a Store but the router is implemented as a different GRPC server struct. -mod key_image_router_service; +//Router & store system. KeyImageService can function as a Store but the router +// is implemented as a different GRPC server struct. mod key_image_router_server; +mod key_image_router_service; pub use block_service::BlockService; pub use config::LedgerServerConfig; @@ -23,4 +24,4 @@ pub use server::LedgerServer; pub use untrusted_tx_out_service::UntrustedTxOutService; pub use config::LedgerRouterConfig; -pub use key_image_router_server::KeyImageRouterServer; \ No newline at end of file +pub use key_image_router_server::KeyImageRouterServer; diff --git a/fog/ledger/server/src/router_handlers.rs b/fog/ledger/server/src/router_handlers.rs index 411f488cc9..2fe5a7c170 100644 --- a/fog/ledger/server/src/router_handlers.rs +++ b/fog/ledger/server/src/router_handlers.rs @@ -1,20 +1,23 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use crate::{ - error::{router_server_err_to_rpc_status, RouterServerError}, -}; -use futures::{future::try_join_all, TryStreamExt, SinkExt}; -use grpcio::{DuplexSink, RequestStream, RpcStatus, WriteFlags}; +use crate::error::{router_server_err_to_rpc_status, RouterServerError}; +use futures::{future::try_join_all, SinkExt, TryStreamExt}; +use grpcio::{ChannelBuilder, DuplexSink, RequestStream, RpcStatus, WriteFlags}; use mc_attest_api::attest; -use mc_common::{logger::Logger}; +use mc_attest_enclave_api::{ClientSession, EnclaveMessage}; +use mc_common::{logger::Logger, ResponderId}; use mc_fog_api::{ - ledger::{LedgerRequest, LedgerResponse, MultiKeyImageStoreRequest, MultiKeyImageStoreResponse}, ledger_grpc::KeyImageStoreApiClient, + ledger::{ + LedgerRequest, LedgerResponse, MultiKeyImageStoreRequest, MultiKeyImageStoreResponse, + MultiKeyImageStoreResponseStatus, + }, + ledger_grpc::KeyImageStoreApiClient, }; use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::KeyImageStoreUri; +use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; //use mc_fog_ledger_enclave_api::LedgerEnclaveProxy; -use mc_util_grpc::{rpc_invalid_arg_error}; -use std::{str::FromStr, sync::Arc}; +use mc_util_grpc::{rpc_invalid_arg_error, ConnectionUriGrpcioChannel}; +use std::{collections::BTreeMap, str::FromStr, sync::Arc}; #[allow(dead_code)] //FIXME const RETRY_COUNT: usize = 3; @@ -62,9 +65,16 @@ where { if request.has_auth() { handle_auth_request(enclave, request.take_auth(), logger) - } else if request.has_check_key_images() { - handle_query_request(request.take_check_key_images(), enclave, shard_clients, logger).await - // TODO: Handle other cases here as they are added, such as the merkele proof service. + } else if request.has_check_key_images() + handle_query_request( + request.take_check_key_images(), + enclave, + shard_clients, + logger, + ) + .await + // TODO: Handle other cases here as they are added, such as the merkele + // proof service. } else { let rpc_status = rpc_invalid_arg_error( "Inavlid LedgerRequest request", @@ -75,7 +85,8 @@ where } } -/// The result of processing the MultiLedgerStoreQueryResponse from each Fog Ledger Shard. +/// The result of processing the MultiLedgerStoreQueryResponse from each Fog +/// Ledger Shard. pub struct ProcessedShardResponseData { /// gRPC clients for Shards that need to be retried for a successful /// response. @@ -87,14 +98,14 @@ pub struct ProcessedShardResponseData { pub store_uris_for_authentication: Vec, /// New, successfully processed query responses. - pub new_query_responses: Vec, + pub new_query_responses: Vec<(ResponderId, attest::Message)>, } impl ProcessedShardResponseData { pub fn new( shard_clients_for_retry: Vec>, store_uris_for_authentication: Vec, - new_query_responses: Vec, + new_query_responses: Vec<(ResponderId, attest::Message)>, ) -> Self { ProcessedShardResponseData { shard_clients_for_retry, @@ -116,13 +127,18 @@ pub fn process_shard_responses( // We did not receive a query_response for this shard.Therefore, we need to: // (a) retry the query // (b) authenticate with the Ledger Store that returned the decryption_error - if response.has_decryption_error() { - shard_clients_for_retry.push(shard_client); - let store_uri = - KeyImageStoreUri::from_str(&response.get_decryption_error().store_uri)?; - store_uris_for_authentication.push(store_uri); - } else { - new_query_responses.push(response.take_query_response()); + let store_uri = KeyImageStoreUri::from_str(response.get_fog_ledger_store_uri())?; + match response.get_status() { + MultiKeyImageStoreResponseStatus::SUCCESS => { + let store_responder_id = store_uri.responder_id()?; + new_query_responses.push((store_responder_id, response.take_query_response())); + } + MultiKeyImageStoreResponseStatus::AUTHENTICATION_ERROR => { + shard_clients_for_retry.push(shard_client); + store_uris_for_authentication.push(store_uri); + } + // This call will be retried as part of the larger retry logic + MultiKeyImageStoreResponseStatus::NOT_READY => (), } } @@ -151,7 +167,7 @@ where Ok(response) } -#[allow(unused_variables)] // FIXME when enclave code is set up. +#[allow(unused_variables)] // FIXME when enclave code is set up. /// Handles a client's query request. async fn handle_query_request( query: attest::Message, @@ -162,13 +178,29 @@ async fn handle_query_request( where E: LedgerEnclaveProxy, { - let mut query_responses: Vec = Vec::with_capacity(shard_clients.len()); + let mut query_responses: BTreeMap> = BTreeMap::new(); let mut shard_clients = shard_clients.clone(); - // TODO: use retry crate? - for _ in 0..RETRY_COUNT { - /* + let sealed_query = enclave + .decrypt_and_seal_query(query.into()) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: internal encryption error", + err.into(), + logger.clone(), + ) + })?; + + // The retry logic here is: + // Set retries remaining to RETRY_COUNT + // Send query and process responses + // If there's a response from every shard, we're done + // If there's a new store, repeat + // If there's no new store and we don't have enough responses, decrement + // RETRY_COUNT and loop + let mut remaining_retries = RETRY_COUNT; + while remaining_retries > 0 { let multi_ledger_store_query_request = enclave - .create_multi_key_image_store_query_data(query.clone().into()) + .create_multi_key_image_store_query_data(sealed_query.clone()) .map_err(|err| { router_server_err_to_rpc_status( "Query: internal encryption error", @@ -176,10 +208,9 @@ where logger.clone(), ) })? - .into();*/ - let test_request = MultiKeyImageStoreRequest::default(); + .into(); let clients_and_responses = - route_query(&test_request, shard_clients.clone()) + route_query(&multi_ledger_store_query_request, shard_clients.clone()) .await .map_err(|err| { router_server_err_to_rpc_status( @@ -189,36 +220,62 @@ where ) })?; - let mut processed_shard_response_data = process_shard_responses( - clients_and_responses, - ) - .map_err(|err| { - router_server_err_to_rpc_status( - "Query: internal query response processing", - err, - logger.clone(), - ) - })?; + let processed_shard_response_data = process_shard_responses(clients_and_responses) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: internal query response processing", + err, + logger.clone(), + ) + })?; - query_responses.append(&mut processed_shard_response_data.new_query_responses); - shard_clients = processed_shard_response_data.shard_clients_for_retry; - if shard_clients.is_empty() { + for (store_responder_id, new_query_response) in processed_shard_response_data + .new_query_responses + .into_iter() + { + query_responses.insert(store_responder_id, new_query_response.into()); + } + + if query_responses.len() >= shard_clients.len() { break; } - /* TODO pending ledger router code enclave-side. - - authenticate_ledger_stores( - enclave.clone(), - processed_shard_response_data.store_uris_for_authentication, + shard_clients = processed_shard_response_data.shard_clients_for_retry; + if !shard_clients.is_empty() { + authenticate_ledger_stores( + enclave.clone(), + processed_shard_response_data.store_uris_for_authentication, + logger.clone(), + ) + .await?; + } else { + remaining_retries -= 1; + } + } + + if remaining_retries == 0 { + return Err(router_server_err_to_rpc_status( + "Query: timed out connecting to key image stores", + RouterServerError::LedgerStoreError(format!( + "Received {} responses which failed to advance the MultiKeyImageStoreRequest", + RETRY_COUNT + )), logger.clone(), - ) - .await?;*/ + )); } - // TODO: Collate the query_responses into one response for the client. Make an - // enclave method for this. - let response = LedgerResponse::new(); + let query_response = enclave + .collate_shard_query_responses(sealed_query, query_responses) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: shard response collation", + RouterServerError::Enclave(err), + logger.clone(), + ) + })?; + + let mut response = LedgerResponse::new(); + response.set_check_key_image_response(query_response.into()); Ok(response) } @@ -244,16 +301,12 @@ async fn query_shard( Ok((shard_client, response)) } - - -/* TODO pending ledger router code enclave-side. - // Authenticates Fog Ledger Stores that have previously not been authenticated. async fn authenticate_ledger_stores( enclave: E, - ledger_store_uris: Vec, + ledger_store_uris: Vec, logger: Logger, -) -> Result<(), RpcStatus> { +) -> Result, RpcStatus> { let pending_auth_requests = ledger_store_uris .into_iter() .map(|store_uri| authenticate_ledger_store(enclave.clone(), store_uri, logger.clone())); @@ -271,25 +324,26 @@ async fn authenticate_ledger_stores( // Authenticates a Fog Ledger Store that has previously not been authenticated. async fn authenticate_ledger_store( enclave: E, - ledger_store_url: LedgerStoreUri, + ledger_store_url: KeyImageStoreUri, logger: Logger, ) -> Result<(), RouterServerError> { let ledger_store_id = ResponderId::from_str(&ledger_store_url.to_string())?; - let client_auth_request = enclave.ledger_store_init(ledger_store_id.clone())?; + let client_auth_request = enclave.connect_to_key_image_store(ledger_store_id.clone())?; let grpc_env = Arc::new( grpcio::EnvBuilder::new() .name_prefix("authenticate-ledger-store".to_string()) .build(), ); let ledger_store_client = KeyImageStoreApiClient::new( - ChannelBuilder::default_channel_builder(grpc_env).connect_to_uri(&ledger_store_url, &logger), + ChannelBuilder::default_channel_builder(grpc_env) + .connect_to_uri(&ledger_store_url, &logger), ); let auth_unary_receiver = ledger_store_client.auth_async(&client_auth_request.into())?; let auth_response = auth_unary_receiver.await?; - let result = enclave.ledger_store_connect(ledger_store_id, auth_response.into())?; + let result = + enclave.finish_connecting_to_key_image_store(ledger_store_id, auth_response.into())?; Ok(result) } -*/ \ No newline at end of file diff --git a/fog/ledger/test_infra/src/lib.rs b/fog/ledger/test_infra/src/lib.rs index 8f59681a69..1faf74e745 100644 --- a/fog/ledger/test_infra/src/lib.rs +++ b/fog/ledger/test_infra/src/lib.rs @@ -85,7 +85,10 @@ impl LedgerEnclave for MockEnclave { unimplemented!() } - fn connect_to_key_image_store(&self, _ledger_store_id: ResponderId) -> EnclaveResult { + fn connect_to_key_image_store( + &self, + _ledger_store_id: ResponderId, + ) -> EnclaveResult { unimplemented!() } @@ -105,7 +108,7 @@ impl LedgerEnclave for MockEnclave { } fn handle_key_image_store_request( - &self, + &self, _router_query: EnclaveMessage, ) -> EnclaveResult> { unimplemented!() diff --git a/fog/types/src/common.rs b/fog/types/src/common.rs index 8d1bb7ce22..e0591c60c4 100644 --- a/fog/types/src/common.rs +++ b/fog/types/src/common.rs @@ -1,5 +1,7 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation +use alloc::{format, string::String, vec::Vec}; +use core::str::FromStr; use prost::Message; use serde::{Deserialize, Serialize}; @@ -45,9 +47,31 @@ impl core::fmt::Display for BlockRange { } } +impl FromStr for BlockRange { + type Err = String; + + fn from_str(s: &str) -> Result { + let block_indices: Vec = s + .split(',') + .map(|index_str| index_str.trim().parse()) + .collect::, _>>() + .map_err(|_| "BlockRange index is not a number.")?; + if block_indices.len() != 2 { + return Err(format!( + "Block range is composed of two indices, found {} indices", + block_indices.len() + )); + } + let result = BlockRange::new(block_indices[0], block_indices[1]); + + Ok(result) + } +} + #[cfg(test)] mod tests { use super::*; + #[test] fn test_contains() { let range = BlockRange::new(10, 13); @@ -87,4 +111,55 @@ mod tests { assert!(!range.overlaps(&BlockRange::new(0, 10))); assert!(!range.overlaps(&BlockRange::new(13, 100))); } + + #[test] + fn from_string_well_formatted_creates_block_range() { + let start_block = 0; + let end_block = 10; + let block_range_str = format!("{},{}", start_block, end_block); + + let result = BlockRange::from_str(&block_range_str); + + assert!(result.is_ok()); + let block_range = result.unwrap(); + assert_eq!(block_range.start_block, start_block); + assert_eq!(block_range.end_block, end_block); + } + + #[test] + fn from_string_well_formatted_with_whitespace_creates_block_range() { + let start_block = 0; + let end_block = 10; + let block_range_str = format!(" {} , {} ", start_block, end_block); + + let result = BlockRange::from_str(&block_range_str); + + assert!(result.is_ok()); + let block_range = result.unwrap(); + assert_eq!(block_range.start_block, start_block); + assert_eq!(block_range.end_block, end_block); + } + + #[test] + fn from_string_multiple_indices_errors() { + let start_block = 0; + let end_block = 10; + let third_block = 10; + let block_range_str = format!("{},{},{}", start_block, end_block, third_block); + + let result = BlockRange::from_str(&block_range_str); + + assert!(result.is_err()); + } + + #[test] + fn from_string_non_numbers_errors() { + let start_block = 'a'; + let end_block = 'b'; + let block_range_str = format!("{},{}", start_block, end_block); + + let result = BlockRange::from_str(&block_range_str); + + assert!(result.is_err()); + } } diff --git a/fog/uri/src/lib.rs b/fog/uri/src/lib.rs index 89e0cd1a1d..1a2449cb47 100644 --- a/fog/uri/src/lib.rs +++ b/fog/uri/src/lib.rs @@ -18,6 +18,20 @@ impl UriScheme for FogViewRouterScheme { const DEFAULT_INSECURE_PORT: u16 = 3225; } +/// Fog View Router Admin Scheme +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct FogViewRouterAdminScheme {} + +impl UriScheme for FogViewRouterAdminScheme { + /// The part before the '://' of a URL. + const SCHEME_SECURE: &'static str = "fog-view-router-admin"; + const SCHEME_INSECURE: &'static str = "insecure-fog-view-router-admin"; + + /// Default port numbers + const DEFAULT_SECURE_PORT: u16 = 443; + const DEFAULT_INSECURE_PORT: u16 = 3225; +} + /// Fog View Uri Scheme #[derive(Debug, Hash, Ord, PartialOrd, Eq, PartialEq, Clone)] pub struct FogViewScheme {} @@ -121,9 +135,11 @@ pub type FogIngestUri = Uri; /// Uri used when talking to fog-ledger service, with the right default ports /// and scheme. pub type FogLedgerUri = Uri; -/// Uri used when talking to fog ledger router service. +/// Uri used when talking to fog view router admin service. +pub type FogViewRouterAdminUri = Uri; +/// Uri used when talking to fog ledger router service. pub type KeyImageRouterUri = Uri; -/// Uri used when talking to fog ledger store service. +/// Uri used when talking to fog ledger store service. pub type KeyImageStoreUri = Uri; /// Uri used when talking to fog view router service. pub type FogViewRouterUri = Uri; @@ -204,6 +220,17 @@ mod tests { ); assert!(!uri.use_tls()); + let uri = FogViewRouterAdminUri::from_str( + "insecure-fog-view-router-admin://node1.test.mobilecoin.com:3225/", + ) + .unwrap(); + assert_eq!(uri.addr(), "node1.test.mobilecoin.com:3225"); + assert_eq!( + uri.responder_id().unwrap(), + ResponderId::from_str("node1.test.mobilecoin.com:3225").unwrap() + ); + assert!(!uri.use_tls()); + let uri = FogViewStoreUri::from_str("insecure-fog-view-store://node1.test.mobilecoin.com:3225/") .unwrap(); diff --git a/fog/view/connection/Cargo.toml b/fog/view/connection/Cargo.toml index 8a6233625b..503d269ad8 100644 --- a/fog/view/connection/Cargo.toml +++ b/fog/view/connection/Cargo.toml @@ -6,15 +6,21 @@ edition = "2021" license = "GPL-3.0" [dependencies] + +# third-party +aes-gcm = "0.9.4" +futures = "0.3" +grpcio = "0.10.3" + # mobilecoin +mc-attest-ake = { path = "../../../attest/ake" } mc-attest-api = { path = "../../../attest/api" } mc-attest-core = { path = "../../../attest/core" } mc-attest-verifier = { path = "../../../attest/verifier" } mc-common = { path = "../../../common", features = ["log"] } mc-crypto-keys = { path = "../../../crypto/keys" } -mc-util-grpc = { path = "../../../util/grpc" } -mc-util-serial = { path = "../../../util/serial" } -mc-util-telemetry = { path = "../../../util/telemetry" } +mc-crypto-noise = { path = "../../../crypto/noise" } +mc-crypto-rand = { path = "../../../crypto/rand" } # fog mc-fog-api = { path = "../../api" } @@ -22,9 +28,10 @@ mc-fog-enclave-connection = { path = "../../enclave_connection" } mc-fog-types = { path = "../../types" } mc-fog-uri = { path = "../../uri" } mc-fog-view-protocol = { path = "../protocol" } - -# third-party -futures = "0.3" -grpcio = "0.10.3" +mc-util-grpc = { path = "../../../util/grpc" } +mc-util-serial = { path = "../../../util/serial" } +mc-util-telemetry = { path = "../../../util/telemetry" } +mc-util-uri = { path = "../../../util/uri" } retry = "1.3" +sha2 = { version = "0.10", default-features = false } tokio = { version = "1.19.2", features = ["full"] } diff --git a/fog/view/connection/src/bin/router_client.rs b/fog/view/connection/src/bin/router_client.rs deleted file mode 100644 index c095e750b1..0000000000 --- a/fog/view/connection/src/bin/router_client.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -//! A simple binary that creates a Fog View Router Client and sends off -//! requests to the Fog View Router server. - -use mc_fog_uri::FogViewRouterUri; -use mc_fog_view_connection::fog_view_router_client::FogViewRouterGrpcClient; -use std::{str::FromStr, sync::Arc}; - -#[tokio::main(flavor = "multi_thread")] -async fn main() -> Result<(), grpcio::Error> { - let fog_view_router_uri = FogViewRouterUri::from_str("insecure-fog-view-router://127.0.0.1/") - .expect("failed to connect to fog view router uri"); - let env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("Main-RPC".to_string()) - .build(), - ); - let (logger, _global_logger_guard) = - mc_common::logger::create_app_logger(mc_common::logger::o!()); - - let fog_view_router_client = - FogViewRouterGrpcClient::new(fog_view_router_uri.clone(), env.clone(), logger.clone()); - - fog_view_router_client.request().await -} diff --git a/fog/view/connection/src/fog_view_router_client.rs b/fog/view/connection/src/fog_view_router_client.rs index ca3776cf01..df20a9f912 100644 --- a/fog/view/connection/src/fog_view_router_client.rs +++ b/fog/view/connection/src/fog_view_router_client.rs @@ -1,69 +1,249 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -//! Makes dummy requests to the fog view router service +//! Makes requests to the fog view router service +use aes_gcm::Aes256Gcm; use futures::{SinkExt, TryStreamExt}; -use grpcio::{ChannelBuilder, Environment}; -use mc_attest_api::attest::Message; +use grpcio::{ChannelBuilder, ClientDuplexReceiver, ClientDuplexSender, Environment}; +use mc_attest_ake::{ + AuthResponseInput, ClientInitiate, Error as AttestAkeError, Ready, Start, Transition, +}; +use mc_attest_api::attest::{AuthMessage, Message}; +use mc_attest_core::VerificationReport; +use mc_attest_verifier::Verifier; use mc_common::logger::{log, o, Logger}; -use mc_fog_api::{view::FogViewRouterRequest, view_grpc::FogViewRouterApiClient}; -use mc_fog_uri::FogViewRouterUri; +use mc_crypto_keys::X25519; +use mc_crypto_noise::CipherError; +use mc_crypto_rand::McRng; +use mc_fog_api::{ + view::{FogViewRouterRequest, FogViewRouterResponse}, + view_grpc::FogViewRouterApiClient, +}; +use mc_fog_types::view::{QueryRequest, QueryRequestAAD, QueryResponse}; +use mc_fog_uri::{ConnectionUri, FogViewRouterUri}; use mc_util_grpc::ConnectionUriGrpcioChannel; +use mc_util_serial::DecodeError; +use mc_util_uri::UriConversionError; +use sha2::Sha512; use std::sync::Arc; /// A high-level object mediating requests to the fog view router service pub struct FogViewRouterGrpcClient { - /// The fog view router grpc client - fog_view_router_client: FogViewRouterApiClient, /// A logger object logger: Logger, + + /// The AKE state machine object, if one is available. + attest_cipher: Option>, + + _fog_view_router_client: FogViewRouterApiClient, + + /// Sends requests to the fog view router + request_sender: ClientDuplexSender, + + /// Receives responses from the fog view router + response_receiver: ClientDuplexReceiver, + + uri: FogViewRouterUri, + + /// An object which can verify a fog node's provided IAS report + verifier: Verifier, } impl FogViewRouterGrpcClient { - /// Creates a new fog view router grpc client + /// Creates a new fog view router grpc client and opens a streaming + /// connection to the fog view router service. /// /// Arguments: /// * uri: The Uri to connect to + /// * verifier: The attestation verifier /// * env: A grpc environment (thread pool) to use for this connection /// * logger: For logging - pub fn new(uri: FogViewRouterUri, env: Arc, logger: Logger) -> Self { + pub fn new( + uri: FogViewRouterUri, + verifier: Verifier, + env: Arc, + logger: Logger, + ) -> Self { let logger = logger.new(o!("mc.fog.view.router.uri" => uri.to_string())); let ch = ChannelBuilder::default_channel_builder(env).connect_to_uri(&uri, &logger); - let fog_view_router_client = FogViewRouterApiClient::new(ch); + let (request_sender, response_receiver) = fog_view_router_client + .request() + .expect("Could not retrieve grpc sender and receiver."); Self { - fog_view_router_client, logger, + attest_cipher: None, + _fog_view_router_client: fog_view_router_client, + request_sender, + response_receiver, + uri, + verifier, } } - /// Makes streaming requests to the fog view router service. - pub async fn request(&self) -> Result<(), grpcio::Error> { - let (mut sink, mut receiver) = self.fog_view_router_client.request()?; - let attested_message = Message::new(); + fn is_attested(&self) -> bool { + self.attest_cipher.is_some() + } + + async fn attest(&mut self) -> Result { + // If we have an existing attestation, nuke it. + self.deattest(); + + let mut csprng = McRng::default(); + + let initiator = Start::new(self.uri.responder_id()?.to_string()); + + let init_input = ClientInitiate::::default(); + let (initiator, auth_request_output) = initiator.try_next(&mut csprng, init_input)?; + + let attested_message: AuthMessage = auth_request_output.into(); let mut request = FogViewRouterRequest::new(); - request.set_query(attested_message); - let send = async move { - for i in 0..5 { - log::info!(self.logger, "Sending message {}", i); - sink.send((request.clone(), grpcio::WriteFlags::default())) - .await?; - } - sink.close().await?; - Ok(()) as Result<(), grpcio::Error> + request.set_auth(attested_message); + self.request_sender + .send((request.clone(), grpcio::WriteFlags::default())) + .await?; + + let mut response = self + .response_receiver + .try_next() + .await? + .ok_or(Error::ResponseNotReceived)?; + let auth_response_msg = response.take_auth(); + + // Process server response, check if key exchange is successful + let auth_response_event = + AuthResponseInput::new(auth_response_msg.into(), self.verifier.clone()); + let (initiator, verification_report) = + initiator.try_next(&mut csprng, auth_response_event)?; + + self.attest_cipher = Some(initiator); + + Ok(verification_report) + } + + fn deattest(&mut self) { + if self.is_attested() { + log::trace!(self.logger, "Tearing down existing attested connection."); + self.attest_cipher = None; + } + } + + /// Makes streaming requests to the fog view router service. + pub async fn query( + &mut self, + start_from_user_event_id: i64, + start_from_block_index: u64, + search_keys: Vec>, + ) -> Result { + log::trace!(self.logger, "Query was called"); + if !self.is_attested() { + let verification_report = self.attest().await; + verification_report?; + } + + let plaintext_request = QueryRequest { + get_txos: search_keys, }; - let receive = async move { - let mut counter = 0; - while (receiver.try_next().await?).is_some() { - counter += 1; - log::info!(self.logger, "Got message {} ", counter); - } - Ok(()) + let req_aad = QueryRequestAAD { + start_from_user_event_id, + start_from_block_index, + }; + + let aad = mc_util_serial::encode(&req_aad); + + let msg = { + let attest_cipher = self + .attest_cipher + .as_mut() + .expect("no enclave_connection even though attest succeeded"); + + let mut msg = Message::new(); + msg.set_channel_id(Vec::from(attest_cipher.binding())); + msg.set_aad(aad.clone()); + + let plaintext_bytes = mc_util_serial::encode(&plaintext_request); + + let request_ciphertext = attest_cipher.encrypt(&aad, &plaintext_bytes)?; + msg.set_data(request_ciphertext); + msg }; - let (sr, rr) = futures::join!(send, receive); - sr.and(rr) + let mut request = FogViewRouterRequest::new(); + request.set_query(msg); + + self.request_sender + .send((request.clone(), grpcio::WriteFlags::default())) + .await?; + + let message = self + .response_receiver + .try_next() + .await? + .ok_or(Error::ResponseNotReceived)? + .take_query(); + + { + let attest_cipher = self + .attest_cipher + .as_mut() + .expect("no enclave_connection even though attest succeeded"); + + let plaintext_bytes = attest_cipher.decrypt(message.get_aad(), message.get_data())?; + let plaintext_response: QueryResponse = mc_util_serial::decode(&plaintext_bytes)?; + Ok(plaintext_response) + } + } +} + +/// Errors related to the Fog View Router Client. +pub enum Error { + /// Decode errors. + Decode(DecodeError), + + /// Uri conversion errors. + UriConversion(UriConversionError), + + /// Cipher errors. + Cipher(CipherError), + + /// Attestation errors. + Attestation(AttestAkeError), + + /// Grpc errors. + Grpc(grpcio::Error), + + /// Response not received + ResponseNotReceived, +} + +impl From for Error { + fn from(err: DecodeError) -> Self { + Self::Decode(err) + } +} + +impl From for Error { + fn from(err: CipherError) -> Self { + Self::Cipher(err) + } +} + +impl From for Error { + fn from(err: grpcio::Error) -> Self { + Self::Grpc(err) + } +} + +impl From for Error { + fn from(err: UriConversionError) -> Self { + Self::UriConversion(err) + } +} + +impl From for Error { + fn from(err: AttestAkeError) -> Self { + Self::Attestation(err) } } diff --git a/fog/view/enclave/Cargo.toml b/fog/view/enclave/Cargo.toml index 395dd25975..6476bf96cc 100644 --- a/fog/view/enclave/Cargo.toml +++ b/fog/view/enclave/Cargo.toml @@ -11,6 +11,7 @@ mc-attest-core = { path = "../../../attest/core" } mc-attest-enclave-api = { path = "../../../attest/enclave-api" } mc-attest-verifier = { path = "../../../attest/verifier" } mc-common = { path = "../../../common", features = ["log"] } +mc-crypto-ake-enclave = { path = "../../../crypto/ake/enclave" } mc-crypto-keys = { path = "../../../crypto/keys" } mc-enclave-boundary = { path = "../../../enclave-boundary" } mc-sgx-debug-edl = { path = "../../../sgx/debug-edl" } diff --git a/fog/view/enclave/api/Cargo.toml b/fog/view/enclave/api/Cargo.toml index bf76efa5f7..abc64e9f1c 100644 --- a/fog/view/enclave/api/Cargo.toml +++ b/fog/view/enclave/api/Cargo.toml @@ -10,6 +10,7 @@ license = "GPL-3.0" mc-attest-core = { path = "../../../../attest/core", default-features = false } mc-attest-enclave-api = { path = "../../../../attest/enclave-api", default-features = false } mc-common = { path = "../../../../common", default-features = false } +mc-crypto-ake-enclave = { path = "../../../../crypto/ake/enclave", default-features = false } mc-crypto-keys = { path = "../../../../crypto/keys", default-features = false } mc-crypto-noise = { path = "../../../../crypto/noise", default-features = false } mc-sgx-compat = { path = "../../../../sgx/compat", default-features = false } diff --git a/fog/view/enclave/api/src/lib.rs b/fog/view/enclave/api/src/lib.rs index 97ee49f20b..edc1fc2e11 100644 --- a/fog/view/enclave/api/src/lib.rs +++ b/fog/view/enclave/api/src/lib.rs @@ -8,13 +8,13 @@ extern crate alloc; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; use core::result::Result as StdResult; use displaydoc::Display; use mc_attest_core::{Quote, Report, SgxError, TargetInfo, VerificationReport}; use mc_attest_enclave_api::{ ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, - Error as AttestEnclaveError, + Error as AttestEnclaveError, SealedClientMessage, }; use mc_common::ResponderId; use mc_crypto_keys::X25519Public; @@ -84,15 +84,24 @@ pub enum ViewEnclaveRequest { Query(EnclaveMessage, UntrustedQueryResponse), /// Request from untrusted to add encrypted tx out records to ORAM AddRecords(Vec), - /// Takes an encrypted fog_types::view::QueryRequest and returns a list of + /// Takes a client query message and returns a SealedClientMessage + /// sealed for the current enclave. + DecryptAndSealQuery(EnclaveMessage), + /// Takes a sealed fog_types::view::QueryRequest and returns a list of /// fog_types::view::QueryRequest. - CreateMultiViewStoreQuery(EnclaveMessage), + CreateMultiViewStoreQuery(SealedClientMessage), /// Begin a client connection to a Fog View Store discovered after /// initialization. ViewStoreInit(ResponderId), /// Complete the client connection to a Fog View store that accepted our /// client auth request. This is meant to be called after [ViewStoreInit]. ViewStoreConnect(ResponderId, ClientAuthResponse), + /// Collates shard query responses into a single query response for the + /// client. + CollateQueryResponses( + SealedClientMessage, + BTreeMap>, + ), } /// The parameters needed to initialize the view enclave @@ -158,14 +167,30 @@ pub trait ViewEnclaveApi: ReportableEnclave { /// enclave's ORAM fn add_records(&self, records: Vec) -> Result<()>; + /// Decrypts a client query message and converts it into a + /// SealedClientMessage which can be unsealed multiple times to + /// construct the MultiViewStoreQuery. + fn decrypt_and_seal_query( + &self, + client_query: EnclaveMessage, + ) -> Result; + /// Transforms a client query request into a list of query request data. /// /// The returned list is meant to be used to construct the /// MultiViewStoreQuery, which is sent to each shard. fn create_multi_view_store_query_data( &self, - client_query: EnclaveMessage, + sealed_query: SealedClientMessage, ) -> Result>>; + + /// Receives all of the shards' query responses and collates them into one + /// query response for the client. + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result>; } /// Helper trait which reduces boiler-plate in untrusted side @@ -221,6 +246,8 @@ pub enum Error { EnclaveNotInitialized, /// Cipher encryption failed: {0} Cipher(CipherError), + /// Fog View Shard query response collation error. + QueryResponseCollation, } impl From for Error { diff --git a/fog/view/enclave/impl/Cargo.toml b/fog/view/enclave/impl/Cargo.toml index 0d1ddccc00..9ce1bc1bf3 100644 --- a/fog/view/enclave/impl/Cargo.toml +++ b/fog/view/enclave/impl/Cargo.toml @@ -6,7 +6,8 @@ edition = "2021" license = "GPL-3.0" [dependencies] -# mobilecoin +aes-gcm = "0.9.4" +aligned-cmov = "2.2" mc-attest-ake = { path = "../../../../attest/ake", default-features = false } mc-attest-core = { path = "../../../../attest/core", default-features = false } mc-attest-enclave-api = { path = "../../../../attest/enclave-api", default-features = false } @@ -14,23 +15,16 @@ mc-common = { path = "../../../../common", default-features = false } mc-crypto-ake-enclave = { path = "../../../../crypto/ake/enclave" } mc-crypto-keys = { path = "../../../../crypto/keys", default-features = false } mc-crypto-rand = { path = "../../../../crypto/rand", default-features = false } -mc-sgx-compat = { path = "../../../../sgx/compat", default-features = false } -mc-sgx-report-cache-api = { path = "../../../../sgx/report-cache/api" } -mc-util-serial = { path = "../../../../util/serial", default-features = false } - -# mc-oblivious -aligned-cmov = "2.2" -mc-oblivious-map = "2.2" -mc-oblivious-ram = "2.2" -mc-oblivious-traits = "2.2" - -# fog mc-fog-recovery-db-iface = { path = "../../../recovery_db_iface" } mc-fog-types = { path = "../../../types" } mc-fog-view-enclave-api = { path = "../api" } - -# third-party -aes-gcm = "0.9.4" +mc-oblivious-map = "2.2" +mc-oblivious-ram = "2.2" +mc-oblivious-traits = "2.2" +mc-sgx-compat = { path = "../../../../sgx/compat", default-features = false } +mc-sgx-report-cache-api = { path = "../../../../sgx/report-cache/api" } +mc-util-serial = { path = "../../../../util/serial", default-features = false } [dev-dependencies] +itertools = "0.10.3" mc-common = { path = "../../../../common", features = ["loggers"] } diff --git a/fog/view/enclave/impl/src/lib.rs b/fog/view/enclave/impl/src/lib.rs index 5f5a40a6dc..fefacd0f45 100644 --- a/fog/view/enclave/impl/src/lib.rs +++ b/fog/view/enclave/impl/src/lib.rs @@ -7,11 +7,16 @@ extern crate alloc; mod e_tx_out_store; +mod oblivious_utils; + +use alloc::collections::BTreeMap; use e_tx_out_store::{ETxOutStore, StorageDataSize, StorageMetaSize}; use alloc::vec::Vec; use mc_attest_core::{IasNonce, Quote, QuoteNonce, Report, TargetInfo, VerificationReport}; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; use mc_common::{ logger::{log, Logger}, ResponderId, @@ -20,7 +25,7 @@ use mc_crypto_ake_enclave::{AkeEnclaveState, NullIdentity}; use mc_crypto_keys::X25519Public; use mc_fog_recovery_db_iface::FogUserEvent; use mc_fog_types::{ - view::{QueryRequest, QueryResponse}, + view::{QueryRequest, QueryResponse, TxOutSearchResult}, ETxOutRecord, }; use mc_fog_view_enclave_api::{ @@ -189,15 +194,25 @@ where Ok(()) } + /// Decrypts a client query message and converts it into a + /// SealedClientMessage which can be unsealed multiple times to + /// construct the MultiViewStoreQuery. + fn decrypt_and_seal_query( + &self, + client_query: EnclaveMessage, + ) -> Result { + Ok(self.ake.decrypt_client_message_for_enclave(client_query)?) + } + /// Takes in a client's query request and returns a list of query requests /// to be sent off to each Fog View Store shard. fn create_multi_view_store_query_data( &self, - client_query: EnclaveMessage, + sealed_query: SealedClientMessage, ) -> Result>> { Ok(self .ake - .reencrypt_client_message_for_backends(client_query)?) + .reencrypt_sealed_message_for_backends(&sealed_query)?) } fn view_store_init(&self, view_store_id: ResponderId) -> Result { @@ -213,4 +228,100 @@ where .ake .backend_connect(view_store_id, view_store_auth_response)?) } + + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result> { + if shard_query_responses.is_empty() { + return Ok(EnclaveMessage::default()); + } + let channel_id = sealed_query.channel_id.clone(); + let client_query_plaintext = self.ake.unseal(&sealed_query)?; + let client_query_request: QueryRequest = mc_util_serial::decode(&client_query_plaintext) + .map_err(|e| { + log::error!(self.logger, "Could not decode client query request: {}", e); + Error::ProstDecode + })?; + + let client_query_response = + self.create_client_query_response(client_query_request, shard_query_responses)?; + let response_plaintext_bytes = mc_util_serial::encode(&client_query_response); + let response = + self.ake + .client_encrypt(&channel_id, &sealed_query.aad, &response_plaintext_bytes)?; + + Ok(response) + } +} + +impl ViewEnclave +where + OSC: ORAMStorageCreator, +{ + fn create_client_query_response( + &self, + client_query_request: QueryRequest, + shard_query_responses: BTreeMap>, + ) -> Result { + let encrypted_shard_query_response = shard_query_responses + .values() + .next() + .expect("Shard query responses must have at least one response.") + .clone(); + let shard_query_response_plaintext = + self.ake.client_decrypt(encrypted_shard_query_response)?; + let mut shard_query_response: QueryResponse = + mc_util_serial::decode(&shard_query_response_plaintext).map_err(|e| { + log::error!(self.logger, "Could not decode shard query response: {}", e); + Error::ProstDecode + })?; + + let shard_query_responses = shard_query_responses + .into_iter() + .map(|(responder_id, enclave_message)| { + let plaintext_bytes = self.ake.backend_decrypt(responder_id, enclave_message)?; + let query_response: QueryResponse = mc_util_serial::decode(&plaintext_bytes)?; + + Ok(query_response) + }) + .collect::>>()?; + + shard_query_response.tx_out_search_results = self.get_collated_tx_out_search_results( + client_query_request, + shard_query_responses.clone(), + )?; + shard_query_response.highest_processed_block_count = + self.get_minimum_highest_processed_block_count(shard_query_responses); + + Ok(shard_query_response) + } + + fn get_collated_tx_out_search_results( + &self, + client_query_request: QueryRequest, + shard_query_responses: Vec, + ) -> Result> { + let plaintext_search_results = shard_query_responses + .into_iter() + .flat_map(|response| response.tx_out_search_results) + .collect::>(); + + oblivious_utils::collate_shard_tx_out_search_results( + client_query_request.get_txos, + plaintext_search_results, + ) + } + + fn get_minimum_highest_processed_block_count( + &self, + shard_query_responses: Vec, + ) -> u64 { + shard_query_responses + .into_iter() + .map(|query_response| query_response.highest_processed_block_count) + .min() + .unwrap_or_default() + } } diff --git a/fog/view/enclave/impl/src/oblivious_utils.rs b/fog/view/enclave/impl/src/oblivious_utils.rs new file mode 100644 index 0000000000..26b8bb7d3d --- /dev/null +++ b/fog/view/enclave/impl/src/oblivious_utils.rs @@ -0,0 +1,637 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +//! Contains methods that allow a Fog View Router enclave to combine all of the +//! Fog View Shard's query responses into one query response that'll be returned +//! for the client. + +use crate::Result; + +use aligned_cmov::{ + subtle::{Choice, ConditionallySelectable, ConstantTimeEq}, + CMov, +}; +use alloc::{vec, vec::Vec}; +use mc_fog_types::view::{TxOutSearchResult, TxOutSearchResultCode}; + +/// The default TxOutSearchResultCode used when collating the shard responses. +/// Warning: Do not change this without careful thought because the logic in +/// the [should_over_write_tx_out_search_result] method assumes that +/// the default code is NotFound. +const DEFAULT_TX_OUT_SEARCH_RESULT_CODE: TxOutSearchResultCode = TxOutSearchResultCode::NotFound; +/// The length of the ciphertext returned to the client. +const CLIENT_CIPHERTEXT_LENGTH: usize = 255; + +#[allow(dead_code)] +pub fn collate_shard_tx_out_search_results( + client_search_keys: Vec>, + shard_tx_out_search_results: Vec, +) -> Result> { + let mut client_tx_out_search_results: Vec = client_search_keys + .iter() + .map(|client_search_key| TxOutSearchResult { + search_key: client_search_key.to_vec(), + result_code: DEFAULT_TX_OUT_SEARCH_RESULT_CODE as u32, + ciphertext: vec![0u8; CLIENT_CIPHERTEXT_LENGTH], + }) + .collect(); + + for shard_tx_out_search_result in shard_tx_out_search_results.iter() { + for client_tx_out_search_result in client_tx_out_search_results.iter_mut() { + maybe_overwrite_tx_out_search_result( + client_tx_out_search_result, + shard_tx_out_search_result, + ); + } + } + + Ok(client_tx_out_search_results) +} + +fn maybe_overwrite_tx_out_search_result( + client_tx_out_search_result: &mut TxOutSearchResult, + shard_tx_out_search_result: &TxOutSearchResult, +) { + let should_overwrite_tx_out_search_result = should_overwrite_tx_out_search_result( + client_tx_out_search_result, + shard_tx_out_search_result, + ); + let shard_ciphertext_length = shard_tx_out_search_result.ciphertext.len(); + + let shard_cipher_text_length_delta = + u8::try_from(CLIENT_CIPHERTEXT_LENGTH - shard_ciphertext_length) + .expect("Shard ciphertext length exceeds bounds"); + // Need to add a 1 because the first byte is reserved for the delta. + assert!( + shard_cipher_text_length_delta >= 1, + "Shard ciphertext has unexpected length" + ); + client_tx_out_search_result.ciphertext[0].conditional_assign( + &shard_cipher_text_length_delta, + should_overwrite_tx_out_search_result, + ); + for idx in 0..shard_ciphertext_length { + // Offset the client ciphertext by 1 because the first byte is reserved for the + // length delta. + client_tx_out_search_result.ciphertext[idx + 1].conditional_assign( + &shard_tx_out_search_result.ciphertext[idx], + should_overwrite_tx_out_search_result, + ); + } + client_tx_out_search_result.result_code.cmov( + should_overwrite_tx_out_search_result, + &shard_tx_out_search_result.result_code, + ); +} + +fn should_overwrite_tx_out_search_result( + client_tx_out_search_result: &TxOutSearchResult, + shard_tx_out_search_result: &TxOutSearchResult, +) -> Choice { + let do_search_keys_match = client_tx_out_search_result + .search_key + .ct_eq(&shard_tx_out_search_result.search_key); + + let client_tx_out_search_result_code = client_tx_out_search_result.result_code; + let shard_tx_out_search_result_code = shard_tx_out_search_result.result_code; + + let client_code_is_found: Choice = + client_tx_out_search_result_code.ct_eq(&(TxOutSearchResultCode::Found as u32)); + let client_code_is_not_found: Choice = + client_tx_out_search_result_code.ct_eq(&(TxOutSearchResultCode::NotFound as u32)); + + let shard_code_is_found: Choice = + shard_tx_out_search_result_code.ct_eq(&(TxOutSearchResultCode::Found as u32)); + + let shard_code_is_retryable_error = + is_code_retryable_error(shard_tx_out_search_result.result_code); + let shard_code_is_bad_search_key = + shard_tx_out_search_result_code.ct_eq(&(TxOutSearchResultCode::BadSearchKey as u32)); + + // We make the same query to several shards and get several responses, and + // this logic determines how we fill the one client response. + // At a high level, we want to prioritize "found" responses, and then "bad + // search key" responses, which means the argument was invalid. After that + // the other two responses are "retriable" errors that the client will retry + // after a backoff. The "not found" response is the default response and + // gets overwritten by any other response. + do_search_keys_match + // Always write a Found code + & (shard_code_is_found + // Write a BadSearchKey code IFF the client code is + // -InternalError, + // -RateLimitedError + // -NotFound + // -BadSearchKey + | (shard_code_is_bad_search_key & !client_code_is_found)) + // Write an InternalError OR RateLimited code IFF the code is NotFound. + | (shard_code_is_retryable_error & client_code_is_not_found) +} + +fn is_code_retryable_error(result_code: u32) -> Choice { + let is_internal_error = result_code.ct_eq(&(TxOutSearchResultCode::InternalError as u32)); + let is_rate_limited = result_code.ct_eq(&(TxOutSearchResultCode::RateLimited as u32)); + + is_internal_error | is_rate_limited +} + +#[cfg(test)] +mod tests { + extern crate std; + + use super::*; + use crate::oblivious_utils::CLIENT_CIPHERTEXT_LENGTH; + use itertools::Itertools; + use std::collections::HashSet; + + fn create_test_tx_out_search_result( + search_key: Vec, + ciphertext_number: u8, + ciphertext_length: usize, + result_code: TxOutSearchResultCode, + ) -> TxOutSearchResult { + TxOutSearchResult { + search_key, + result_code: result_code as u32, + ciphertext: vec![ciphertext_number; ciphertext_length], + } + } + + #[test] + fn should_overwrite_tx_out_search_result_client_not_found_shard_has_tx_out_returns_true() { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::NotFound, + ); + let shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ); + + let result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_bad_search_key_shard_has_tx_out_returns_true() { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::BadSearchKey, + ); + let shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ); + + let result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_has_internal_error_shard_has_tx_out_returns_true( + ) { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::InternalError, + ); + let shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ); + + let result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_has_rate_limited_error_shard_has_tx_out_returns_true( + ) { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + let shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ); + + let result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_client_has_found_never_overwritten_returns_false_unless_shard_finds() + { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ); + + let mut shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::NotFound, + ); + let mut result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::BadSearchKey, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::InternalError, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_has_not_found_shard_has_retryable_error_returns_true( + ) { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::NotFound, + ); + + let mut shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::InternalError, + ); + let mut result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_has_bad_search_key_shard_has_retryable_error_returns_false( + ) { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::BadSearchKey, + ); + + let mut shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::InternalError, + ); + let mut result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_has_bad_search_key() { + let search_key = vec![0u8; 10]; + let shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::BadSearchKey, + ); + + let mut client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + let mut result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(result); + + client_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(result); + } + + #[test] + fn should_overwrite_tx_out_search_result_client_has_retryable_error_shard_has_not_found_returns_true( + ) { + let search_key = vec![0u8; 10]; + let client_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::BadSearchKey, + ); + + let mut shard_tx_out_search_result = create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::InternalError, + ); + let mut result: bool = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + + shard_tx_out_search_result = create_test_tx_out_search_result( + search_key, + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::RateLimited, + ); + result = should_overwrite_tx_out_search_result( + &client_tx_out_search_result, + &shard_tx_out_search_result, + ) + .into(); + assert!(!result); + } + + #[test] + fn collate_shard_query_responses_shards_find_all_tx_outs() { + let client_search_keys: Vec> = (0..10).map(|num| vec![num; 10]).collect(); + let shard_tx_out_search_results: Vec = client_search_keys + .iter() + .map(|search_key| { + create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + TxOutSearchResultCode::Found, + ) + }) + .collect(); + + let result = collate_shard_tx_out_search_results( + client_search_keys.clone(), + shard_tx_out_search_results, + ) + .unwrap(); + + let all_tx_out_found = result.iter().all(|tx_out_search_result| { + tx_out_search_result.result_code == TxOutSearchResultCode::Found as u32 + }); + assert!(all_tx_out_found); + + let result_client_search_keys: HashSet> = HashSet::from_iter( + result + .iter() + .map(|tx_out_search_result| tx_out_search_result.search_key.clone()), + ); + assert_eq!( + result_client_search_keys, + HashSet::from_iter(client_search_keys) + ); + } + + #[test] + fn collate_shard_query_responses_shards_one_not_found() { + let client_search_keys: Vec> = (0..10).map(|num| vec![num; 10]).collect(); + let shard_tx_out_search_results: Vec = client_search_keys + .iter() + .enumerate() + .map(|(i, search_key)| { + let result_code = match i { + 0 => TxOutSearchResultCode::NotFound, + _ => TxOutSearchResultCode::Found, + }; + create_test_tx_out_search_result( + search_key.clone(), + 0, + CLIENT_CIPHERTEXT_LENGTH - 1, + result_code, + ) + }) + .collect(); + + let result = collate_shard_tx_out_search_results( + client_search_keys.clone(), + shard_tx_out_search_results, + ) + .unwrap(); + + let result_client_search_keys: HashSet> = HashSet::from_iter( + result + .iter() + .map(|tx_out_search_result| tx_out_search_result.search_key.clone()), + ); + assert_eq!( + result_client_search_keys, + HashSet::from_iter(client_search_keys) + ); + + let not_found_count = result + .iter() + .filter(|tx_out_search_result| { + tx_out_search_result.result_code == TxOutSearchResultCode::NotFound as u32 + }) + .count(); + assert_eq!(not_found_count, 1); + } + + #[test] + fn collate_shard_query_responses_ciphertext_is_client_ciphertext_length_panics() { + let client_search_keys: Vec> = (0..10).map(|num| vec![num; 10]).collect(); + let shard_tx_out_search_results: Vec = client_search_keys + .iter() + .map(|search_key| TxOutSearchResult { + search_key: search_key.clone(), + result_code: TxOutSearchResultCode::NotFound as u32, + ciphertext: vec![0u8; CLIENT_CIPHERTEXT_LENGTH], + }) + .collect(); + + let result = std::panic::catch_unwind(|| { + collate_shard_tx_out_search_results( + client_search_keys.clone(), + shard_tx_out_search_results, + ) + }); + + assert!(result.is_err()); + } + #[test] + fn collate_shard_query_responses_different_ciphertext_lengths_returns_correct_client_ciphertexts( + ) { + let client_search_keys: Vec> = (0..3).map(|num| vec![num; 10]).collect(); + let ciphertext_values = [28u8, 5u8, 128u8]; + let shard_tx_out_search_results: Vec = client_search_keys + .iter() + .enumerate() + .map(|(idx, search_key)| TxOutSearchResult { + search_key: search_key.clone(), + result_code: TxOutSearchResultCode::Found as u32, + ciphertext: vec![ciphertext_values[idx]; idx + 1], + }) + .collect(); + + let results: Vec = + collate_shard_tx_out_search_results(client_search_keys, shard_tx_out_search_results) + .unwrap() + .into_iter() + // Sort by ciphertext length (ascending) in order to know what each expected result + // should be. + .sorted_by(|a, b| Ord::cmp(&b.ciphertext[0], &a.ciphertext[0])) + .collect(); + + let mut expected_first_result = [0u8; CLIENT_CIPHERTEXT_LENGTH]; + let expected_first_result_delta = (CLIENT_CIPHERTEXT_LENGTH - 1) as u8; + expected_first_result[0] = expected_first_result_delta; + expected_first_result[1] = ciphertext_values[0]; + assert_eq!(results[0].ciphertext, expected_first_result); + + let mut expected_second_result = [0u8; CLIENT_CIPHERTEXT_LENGTH]; + let expected_second_result_delta = (CLIENT_CIPHERTEXT_LENGTH - 2) as u8; + expected_second_result[0] = expected_second_result_delta; + expected_second_result[1] = ciphertext_values[1]; + expected_second_result[2] = ciphertext_values[1]; + assert_eq!(results[1].ciphertext, expected_second_result); + + let mut expected_third_result = [0u8; CLIENT_CIPHERTEXT_LENGTH]; + let expected_third_result_delta = (CLIENT_CIPHERTEXT_LENGTH - 3) as u8; + expected_third_result[0] = expected_third_result_delta; + expected_third_result[1] = ciphertext_values[2]; + expected_third_result[2] = ciphertext_values[2]; + expected_third_result[3] = ciphertext_values[2]; + assert_eq!(results[2].ciphertext, expected_third_result); + } +} diff --git a/fog/view/enclave/src/lib.rs b/fog/view/enclave/src/lib.rs index 2f1c7cadf7..7ed067116e 100644 --- a/fog/view/enclave/src/lib.rs +++ b/fog/view/enclave/src/lib.rs @@ -6,12 +6,14 @@ extern crate mc_fog_ocall_oram_storage_untrusted; -use std::{path, result::Result as StdResult, sync::Arc}; +use std::{collections::BTreeMap, path, result::Result as StdResult, sync::Arc}; use mc_attest_core::{ IasNonce, Quote, QuoteNonce, Report, SgxError, TargetInfo, VerificationReport, }; -use mc_attest_enclave_api::{ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage}; +use mc_attest_enclave_api::{ + ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, SealedClientMessage, +}; use mc_attest_verifier::DEBUG_ENCLAVE; use mc_common::{logger::Logger, ResponderId}; use mc_crypto_keys::X25519Public; @@ -199,12 +201,35 @@ impl ViewEnclaveApi for SgxViewEnclave { mc_util_serial::deserialize(&outbuf[..])? } - fn create_multi_view_store_query_data( + fn decrypt_and_seal_query( &self, client_query: EnclaveMessage, + ) -> Result { + let inbuf = + mc_util_serial::serialize(&ViewEnclaveRequest::DecryptAndSealQuery(client_query))?; + let outbuf = self.enclave_call(&inbuf)?; + mc_util_serial::deserialize(&outbuf[..])? + } + + fn create_multi_view_store_query_data( + &self, + sealed_query: SealedClientMessage, ) -> Result>> { let inbuf = mc_util_serial::serialize(&ViewEnclaveRequest::CreateMultiViewStoreQuery( - client_query, + sealed_query, + ))?; + let outbuf = self.enclave_call(&inbuf)?; + mc_util_serial::deserialize(&outbuf[..])? + } + + fn collate_shard_query_responses( + &self, + sealed_query: SealedClientMessage, + shard_query_responses: BTreeMap>, + ) -> Result> { + let inbuf = mc_util_serial::serialize(&ViewEnclaveRequest::CollateQueryResponses( + sealed_query, + shard_query_responses, ))?; let outbuf = self.enclave_call(&inbuf)?; mc_util_serial::deserialize(&outbuf[..])? diff --git a/fog/view/enclave/trusted/Cargo.lock b/fog/view/enclave/trusted/Cargo.lock index 4f0c8d527a..d279dd1932 100644 --- a/fog/view/enclave/trusted/Cargo.lock +++ b/fog/view/enclave/trusted/Cargo.lock @@ -1135,6 +1135,7 @@ dependencies = [ "mc-attest-core", "mc-attest-enclave-api", "mc-common", + "mc-crypto-ake-enclave", "mc-crypto-keys", "mc-crypto-noise", "mc-fog-recovery-db-iface", diff --git a/fog/view/enclave/trusted/src/lib.rs b/fog/view/enclave/trusted/src/lib.rs index 9c66580fc2..da191ea9b3 100644 --- a/fog/view/enclave/trusted/src/lib.rs +++ b/fog/view/enclave/trusted/src/lib.rs @@ -126,8 +126,14 @@ pub fn ecall_dispatcher(inbuf: &[u8]) -> Result, sgx_status_t> { serialize(&ENCLAVE.query(req, untrusted_query_response)) } ViewEnclaveRequest::AddRecords(records) => serialize(&ENCLAVE.add_records(records)), - ViewEnclaveRequest::CreateMultiViewStoreQuery(client_query) => { - serialize(&ENCLAVE.create_multi_view_store_query_data(client_query)) + ViewEnclaveRequest::DecryptAndSealQuery(client_query) => { + serialize(&ENCLAVE.decrypt_and_seal_query(client_query)) + } + ViewEnclaveRequest::CreateMultiViewStoreQuery(sealed_query) => { + serialize(&ENCLAVE.create_multi_view_store_query_data(sealed_query)) + } + ViewEnclaveRequest::CollateQueryResponses(sealed_query, shard_query_responses) => { + serialize(&ENCLAVE.collate_shard_query_responses(sealed_query, shard_query_responses)) } } .or(Err(sgx_status_t::SGX_ERROR_UNEXPECTED)) diff --git a/fog/view/server/Cargo.toml b/fog/view/server/Cargo.toml index a0d573af0f..98f02caaf3 100644 --- a/fog/view/server/Cargo.toml +++ b/fog/view/server/Cargo.toml @@ -21,25 +21,15 @@ futures = "0.3" grpcio = "0.10.3" hex = "0.4" lazy_static = "1.4" -serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } -serde_json = "1.0" # mobilecoin mc-attest-api = { path = "../../../attest/api" } mc-attest-core = { path = "../../../attest/core" } +mc-attest-enclave-api = { path = "../../../attest/enclave-api" } mc-attest-net = { path = "../../../attest/net" } +mc-blockchain-types = { path = "../../../blockchain/types" } mc-common = { path = "../../../common", features = ["log"] } mc-crypto-keys = { path = "../../../crypto/keys" } -mc-sgx-report-cache-untrusted = { path = "../../../sgx/report-cache/untrusted" } -mc-util-cli = { path = "../../../util/cli" } -mc-util-from-random = { path = "../../../util/from-random" } -mc-util-grpc = { path = "../../../util/grpc" } -mc-util-metered-channel = { path = "../../../util/metered-channel" } -mc-util-metrics = { path = "../../../util/metrics" } -mc-util-parse = { path = "../../../util/parse" } -mc-util-serial = { path = "../../../util/serial" } -mc-util-telemetry = { path = "../../../util/telemetry", features = ["jaeger"] } -mc-util-uri = { path = "../../../util/uri" } # fog mc-fog-api = { path = "../../api" } @@ -50,27 +40,39 @@ mc-fog-types = { path = "../../types" } mc-fog-uri = { path = "../../uri" } mc-fog-view-enclave = { path = "../enclave" } mc-fog-view-enclave-api = { path = "../enclave/api" } +mc-sgx-report-cache-untrusted = { path = "../../../sgx/report-cache/untrusted" } +mc-util-cli = { path = "../../../util/cli" } +mc-util-from-random = { path = "../../../util/from-random" } +mc-util-grpc = { path = "../../../util/grpc" } +mc-util-metered-channel = { path = "../../../util/metered-channel" } +mc-util-metrics = { path = "../../../util/metrics" } +mc-util-parse = { path = "../../../util/parse" } +mc-util-serial = { path = "../../../util/serial" } +mc-util-telemetry = { path = "../../../util/telemetry", features = ["jaeger"] } +mc-util-uri = { path = "../../../util/uri" } +serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } +serde_json = "1.0" [dev-dependencies] -pem = "1.0" -portpicker = "0.1.1" -rand = "0.8" -rand_core = "0.6" -tempdir = "0.3" mc-attest-verifier = { path = "../../../attest/verifier" } mc-blockchain-types = { path = "../../../blockchain/types" } mc-common = { path = "../../../common", features = ["loggers"] } mc-crypto-keys = { path = "../../../crypto/keys" } mc-crypto-x509-test-vectors = { path = "../../../crypto/x509/test-vectors" } -mc-transaction-core = { path = "../../../transaction/core" } -mc-util-encodings = { path = "../../../util/encodings" } -mc-util-serial = { path = "../../../util/serial" } -mc-util-test-helper = { path = "../../../util/test-helper" } -mc-util-uri = { path = "../../../util/uri" } mc-fog-test-infra = { path = "../../test_infra" } mc-fog-types = { path = "../../types" } mc-fog-view-connection = { path = "../connection" } mc-fog-view-enclave-measurement = { path = "../enclave/measurement" } mc-fog-view-protocol = { path = "../protocol" } +mc-transaction-core = { path = "../../../transaction/core" } +mc-util-encodings = { path = "../../../util/encodings" } +mc-util-serial = { path = "../../../util/serial" } +mc-util-test-helper = { path = "../../../util/test-helper" } +mc-util-uri = { path = "../../../util/uri" } +pem = "1.0" +portpicker = "0.1.1" +rand = "0.8" +rand_core = "0.6" +tempdir = "0.3" diff --git a/fog/view/server/src/bin/main.rs b/fog/view/server/src/bin/main.rs index f5aa50c54b..38c3125c07 100644 --- a/fog/view/server/src/bin/main.rs +++ b/fog/view/server/src/bin/main.rs @@ -6,7 +6,7 @@ use mc_attest_net::{Client, RaClient}; use mc_common::{logger::log, time::SystemTimeProvider}; use mc_fog_sql_recovery_db::SqlRecoveryDb; use mc_fog_view_enclave::{SgxViewEnclave, ENCLAVE_FILE}; -use mc_fog_view_server::{config::MobileAcctViewConfig, server::ViewServer}; +use mc_fog_view_server::{config, config::MobileAcctViewConfig, server::ViewServer}; use mc_util_cli::ParserWithBuildInfo; use mc_util_grpc::AdminServer; use std::{env, sync::Arc}; @@ -58,12 +58,15 @@ fn main() { let ias_client = Client::new(&config.ias_api_key).expect("Could not create IAS client"); + let config::ShardingStrategy::Epoch(sharding_strategy) = config.sharding_strategy.clone(); + let mut server = ViewServer::new( config.clone(), sgx_enclave, recovery_db, ias_client, SystemTimeProvider::default(), + sharding_strategy, logger.clone(), ); server.start(); diff --git a/fog/view/server/src/bin/router.rs b/fog/view/server/src/bin/router.rs index 64a29b791e..60b337c40f 100644 --- a/fog/view/server/src/bin/router.rs +++ b/fog/view/server/src/bin/router.rs @@ -3,6 +3,7 @@ //! MobileCoin Fog View Router target use grpcio::ChannelBuilder; +use mc_attest_net::{Client, RaClient}; use mc_common::logger::log; use mc_fog_api::view_grpc::FogViewStoreApiClient; use mc_fog_uri::FogViewStoreUri; @@ -56,8 +57,14 @@ fn main() { fog_view_store_grpc_clients.push(fog_view_store_grpc_client); } - let mut router_server = - FogViewRouterServer::new(config, sgx_enclave, fog_view_store_grpc_clients, logger); + let ias_client = Client::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut router_server = FogViewRouterServer::new( + config, + sgx_enclave, + ias_client, + fog_view_store_grpc_clients, + logger, + ); router_server.start(); loop { diff --git a/fog/view/server/src/config.rs b/fog/view/server/src/config.rs index 3b12ebec33..29a21be277 100644 --- a/fog/view/server/src/config.rs +++ b/fog/view/server/src/config.rs @@ -2,7 +2,7 @@ //! Configuration parameters for the MobileCoin Fog View Node #![deny(missing_docs)] - +use crate::sharding_strategy::EpochShardingStrategy; use clap::Parser; use mc_attest_core::ProviderId; use mc_common::ResponderId; @@ -67,6 +67,33 @@ pub struct MobileAcctViewConfig { /// Postgres config #[clap(flatten)] pub postgres_config: SqlRecoveryDbConnectionConfig, + + /// Determines which group of TxOuts the Fog View Store instance will + /// process. + #[clap(long, default_value = "default")] + pub sharding_strategy: ShardingStrategy, +} + +/// Determines which group of TxOuts the Fog View Store instance will process. +#[derive(Clone, Serialize)] +pub enum ShardingStrategy { + /// URI used by the FogViewServer when fulfilling direct client requests. + Epoch(EpochShardingStrategy), +} + +impl FromStr for ShardingStrategy { + type Err = String; + + fn from_str(s: &str) -> Result { + if s.eq("default") { + return Ok(ShardingStrategy::Epoch(EpochShardingStrategy::default())); + } + if let Ok(epoch_sharding_strategy) = EpochShardingStrategy::from_str(s) { + return Ok(ShardingStrategy::Epoch(epoch_sharding_strategy)); + } + + Err("Invalid sharding strategy config.".to_string()) + } } /// A FogViewServer can either fulfill client requests directly or fulfill Fog @@ -108,6 +135,14 @@ pub struct FogViewRouterConfig { #[clap(long, env = "MC_CLIENT_LISTEN_URI")] pub client_listen_uri: FogViewRouterUri, + /// PEM-formatted keypair to send with an Attestation Request. + #[clap(long, env = "MC_IAS_API_KEY")] + pub ias_api_key: String, + + /// The IAS SPID to use when getting a quote + #[clap(long, env = "MC_IAS_SPID")] + pub ias_spid: ProviderId, + // TODO: Add shard uris which are of type Vec. /// The capacity to build the OMAP (ORAM hash table) with. /// About 75% of this capacity can be used. diff --git a/fog/view/server/src/db_fetcher.rs b/fog/view/server/src/db_fetcher.rs index e969cf8d97..437f0f4028 100644 --- a/fog/view/server/src/db_fetcher.rs +++ b/fog/view/server/src/db_fetcher.rs @@ -2,7 +2,7 @@ //! An object for managing background data fetches from the recovery database. -use crate::{block_tracker::BlockTracker, counters}; +use crate::{block_tracker::BlockTracker, counters, sharding_strategy::ShardingStrategy}; use mc_common::logger::{log, Logger}; use mc_crypto_keys::CompressedRistrettoPublic; use mc_fog_recovery_db_iface::{IngressPublicKeyRecord, IngressPublicKeyRecordFilters, RecoveryDb}; @@ -75,11 +75,16 @@ pub struct DbFetcher { } impl DbFetcher { - pub fn new( + pub fn new( db: DB, readiness_indicator: ReadinessIndicator, + sharding_strategy: SS, logger: Logger, - ) -> Self { + ) -> Self + where + DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, + { let stop_requested = Arc::new(AtomicBool::new(false)); let shared_state = Arc::new(Mutex::new(DbFetcherSharedState::default())); @@ -102,6 +107,7 @@ impl DbFetcher { thread_shared_state, thread_num_queued_records_limiter, readiness_indicator, + sharding_strategy, logger, ) }) @@ -165,25 +171,35 @@ impl Drop for DbFetcher { } } -struct DbFetcherThread { +struct DbFetcherThread +where + DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, +{ db: DB, stop_requested: Arc, shared_state: Arc>, block_tracker: BlockTracker, num_queued_records_limiter: Arc<(Mutex, Condvar)>, readiness_indicator: ReadinessIndicator, + sharding_strategy: SS, logger: Logger, } /// Background worker thread implementation that takes care of periodically /// polling data out of the database. -impl DbFetcherThread { +impl DbFetcherThread +where + DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, +{ pub fn start( db: DB, stop_requested: Arc, shared_state: Arc>, num_queued_records_limiter: Arc<(Mutex, Condvar)>, readiness_indicator: ReadinessIndicator, + sharding_strategy: SS, logger: Logger, ) { let thread = Self { @@ -193,6 +209,7 @@ impl DbFetcherThread { block_tracker: BlockTracker::new(logger.clone()), num_queued_records_limiter, readiness_indicator, + sharding_strategy, logger, }; thread.run(); @@ -305,6 +322,14 @@ impl DbFetcherThread { // Mark that we are done fetching data for this block. self.block_tracker.block_processed(ingress_key, block_index); + if !self.sharding_strategy.should_process_block(block_index) { + log::trace!( + self.logger, + "Not adding block_index {} TxOuts because this shard is not responsible for it.", + block_index, + ); + continue; + } // Store the fetched records so that they could be consumed by the enclave // when its ready. @@ -366,6 +391,7 @@ impl DbFetcherThread { #[cfg(test)] mod tests { use super::*; + use crate::sharding_strategy::EpochShardingStrategy; use mc_attest_core::VerificationReport; use mc_common::logger::test_with_logger; use mc_fog_recovery_db_iface::{IngressPublicKeyStatus, ReportData, ReportDb}; @@ -380,7 +406,12 @@ mod tests { let mut rng: StdRng = SeedableRng::from_seed([123u8; 32]); let db_test_context = SqlRecoveryDbTestContext::new(logger.clone()); let db = db_test_context.get_db_instance(); - let db_fetcher = DbFetcher::new(db.clone(), Default::default(), logger); + let db_fetcher = DbFetcher::new( + db.clone(), + Default::default(), + EpochShardingStrategy::default(), + logger, + ); // Initially, our database starts empty. let ingress_keys = db_fetcher.get_highest_processed_block_context(); @@ -610,7 +641,12 @@ mod tests { let mut rng: StdRng = SeedableRng::from_seed([123u8; 32]); let db_test_context = SqlRecoveryDbTestContext::new(logger.clone()); let db = db_test_context.get_db_instance(); - let db_fetcher = DbFetcher::new(db.clone(), Default::default(), logger); + let db_fetcher = DbFetcher::new( + db.clone(), + Default::default(), + EpochShardingStrategy::default(), + logger, + ); // Register two ingress keys that have some overlap: // key_id1 starts at block 0, key2 starts at block 5. @@ -667,7 +703,12 @@ mod tests { let mut rng: StdRng = SeedableRng::from_seed([123u8; 32]); let db_test_context = SqlRecoveryDbTestContext::new(logger.clone()); let db = db_test_context.get_db_instance(); - let db_fetcher = DbFetcher::new(db.clone(), Default::default(), logger); + let db_fetcher = DbFetcher::new( + db.clone(), + Default::default(), + EpochShardingStrategy::default(), + logger, + ); // Register two ingress keys that have some overlap: // invoc_id1 starts at block 0, invoc_id2 starts at block 50. diff --git a/fog/view/server/src/error.rs b/fog/view/server/src/error.rs index ee6786701a..c4fc627407 100644 --- a/fog/view/server/src/error.rs +++ b/fog/view/server/src/error.rs @@ -33,6 +33,12 @@ impl From for RouterServerError { } } +impl From for RouterServerError { + fn from(src: mc_util_uri::UriConversionError) -> Self { + RouterServerError::ViewStoreError(format!("{}", src)) + } +} + pub fn router_server_err_to_rpc_status( context: &str, src: RouterServerError, diff --git a/fog/view/server/src/fog_view_router_server.rs b/fog/view/server/src/fog_view_router_server.rs index 123c0f56c4..9597f623ac 100644 --- a/fog/view/server/src/fog_view_router_server.rs +++ b/fog/view/server/src/fog_view_router_server.rs @@ -4,28 +4,43 @@ //! Constructible from config (for testability) and with a mechanism for //! stopping it -use crate::{config::FogViewRouterConfig, fog_view_router_service::FogViewRouterService}; +use crate::{config::FogViewRouterConfig, counters, fog_view_router_service::FogViewRouterService}; use futures::executor::block_on; +use mc_attest_net::RaClient; use mc_common::logger::{log, Logger}; use mc_fog_api::view_grpc; use mc_fog_uri::ConnectionUri; use mc_fog_view_enclave::ViewEnclaveProxy; +use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; use std::sync::Arc; -pub struct FogViewRouterServer { +pub struct FogViewRouterServer +where + E: ViewEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ server: grpcio::Server, + enclave: E, + config: FogViewRouterConfig, logger: Logger, + ra_client: RC, + report_cache_thread: Option, } -impl FogViewRouterServer { +impl FogViewRouterServer +where + E: ViewEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ /// Creates a new view router server instance - pub fn new( + pub fn new( config: FogViewRouterConfig, enclave: E, + ra_client: RC, shards: Vec, logger: Logger, - ) -> FogViewRouterServer + ) -> FogViewRouterServer where E: ViewEnclaveProxy, { @@ -38,7 +53,7 @@ impl FogViewRouterServer { ); let fog_view_router_service = view_grpc::create_fog_view_router_api( - FogViewRouterService::new(enclave, shards, logger.clone()), + FogViewRouterService::new(enclave.clone(), shards, logger.clone()), ); log::debug!(logger, "Constructed Fog View Router GRPC Service"); @@ -60,11 +75,28 @@ impl FogViewRouterServer { let server = server_builder.build().unwrap(); - Self { server, logger } + Self { + server, + enclave, + config, + logger, + ra_client, + report_cache_thread: None, + } } /// Starts the server pub fn start(&mut self) { + self.report_cache_thread = Some( + ReportCacheThread::start( + self.enclave.clone(), + self.ra_client.clone(), + self.config.ias_spid, + &counters::ENCLAVE_REPORT_TIMESTAMP, + self.logger.clone(), + ) + .expect("failed starting report cache thread"), + ); self.server.start(); for (host, port) in self.server.bind_addrs() { log::info!(self.logger, "API listening on {}:{}", host, port); @@ -73,11 +105,18 @@ impl FogViewRouterServer { /// Stops the server pub fn stop(&mut self) { + if let Some(ref mut thread) = self.report_cache_thread.take() { + thread.stop().expect("Could not stop report cache thread"); + } block_on(self.server.shutdown()).expect("Could not stop grpc server"); } } -impl Drop for FogViewRouterServer { +impl Drop for FogViewRouterServer +where + E: ViewEnclaveProxy, + RC: RaClient + Send + Sync + 'static, +{ fn drop(&mut self) { self.stop(); } diff --git a/fog/view/server/src/fog_view_service.rs b/fog/view/server/src/fog_view_service.rs index dd8f96cf29..45369f83f3 100644 --- a/fog/view/server/src/fog_view_service.rs +++ b/fog/view/server/src/fog_view_service.rs @@ -1,16 +1,20 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use crate::{config::ClientListenUri, server::DbPollSharedState}; +use crate::{ + config::ClientListenUri, server::DbPollSharedState, sharding_strategy::ShardingStrategy, +}; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use mc_attest_api::attest; use mc_common::logger::{log, Logger}; use mc_fog_api::{ - view::{MultiViewStoreQueryRequest, MultiViewStoreQueryResponse}, + view::{ + MultiViewStoreQueryRequest, MultiViewStoreQueryResponse, MultiViewStoreQueryResponseStatus, + }, view_grpc::{FogViewApi, FogViewStoreApi}, }; use mc_fog_recovery_db_iface::RecoveryDb; use mc_fog_types::view::QueryRequestAAD; -use mc_fog_uri::ConnectionUri; +use mc_fog_uri::{ConnectionUri, FogViewStoreUri}; use mc_fog_view_enclave::{Error as ViewEnclaveError, ViewEnclaveProxy}; use mc_fog_view_enclave_api::UntrustedQueryResponse; use mc_util_grpc::{ @@ -22,7 +26,12 @@ use mc_util_telemetry::{tracer, Tracer}; use std::sync::{Arc, Mutex}; #[derive(Clone)] -pub struct FogViewService { +pub struct FogViewService +where + E: ViewEnclaveProxy, + DB: RecoveryDb + Send + Sync, + SS: ShardingStrategy, +{ /// Enclave providing access to the Recovery DB enclave: E, @@ -40,9 +49,17 @@ pub struct FogViewService { /// Slog logger object logger: Logger, + + /// Dictates what blocks to process. + sharding_strategy: SS, } -impl FogViewService { +impl FogViewService +where + E: ViewEnclaveProxy, + DB: RecoveryDb + Send + Sync, + SS: ShardingStrategy, +{ /// Creates a new fog-view-service node (but does not create sockets and /// start it etc.) pub fn new( @@ -51,6 +68,7 @@ impl FogViewService { db_poll_shared_state: Arc>, authenticator: Arc, client_listen_uri: ClientListenUri, + sharding_strategy: SS, logger: Logger, ) -> Self { Self { @@ -59,6 +77,7 @@ impl FogViewService { db_poll_shared_state, authenticator, client_listen_uri, + sharding_strategy, logger, } } @@ -151,6 +170,37 @@ impl FogViewService { }) } + fn process_queries( + &mut self, + fog_view_store_uri: FogViewStoreUri, + queries: Vec, + ) -> MultiViewStoreQueryResponse { + let mut response = MultiViewStoreQueryResponse::new(); + response.set_fog_view_store_uri(fog_view_store_uri.url().to_string()); + for query in queries.into_iter() { + let result = self.query_impl(query); + // Only one of the query messages in an MVSQR is intended for this store + if let Ok(attested_message) = result { + { + let shared_state = self.db_poll_shared_state.lock().expect("mutex poisoned"); + if !self + .sharding_strategy + .is_ready_to_serve_tx_outs(shared_state.processed_block_count.into()) + { + response.set_status(MultiViewStoreQueryResponseStatus::NOT_READY); + } else { + response.set_query_response(attested_message); + response.set_status(MultiViewStoreQueryResponseStatus::SUCCESS); + } + } + return response; + } + } + + response.set_status(MultiViewStoreQueryResponseStatus::AUTHENTICATION_ERROR); + response + } + // Helper function that is common fn enclave_err_to_rpc_status(&self, context: &str, src: ViewEnclaveError) -> RpcStatus { // Treat prost-decode error as an invalid arg, @@ -169,7 +219,12 @@ impl FogViewService { } // Implement grpc trait -impl FogViewApi for FogViewService { +impl FogViewApi for FogViewService +where + E: ViewEnclaveProxy, + DB: RecoveryDb + Send + Sync, + SS: ShardingStrategy, +{ fn auth( &mut self, ctx: RpcContext, @@ -204,7 +259,12 @@ impl FogViewApi for FogViewSe } /// Implement the FogViewStoreService gRPC trait. -impl FogViewStoreApi for FogViewService { +impl FogViewStoreApi for FogViewService +where + E: ViewEnclaveProxy, + DB: RecoveryDb + Send + Sync, + SS: ShardingStrategy, +{ fn auth( &mut self, ctx: RpcContext, @@ -235,20 +295,7 @@ impl FogViewStoreApi for FogV return send_result(ctx, sink, err.into(), logger); } if let ClientListenUri::Store(fog_view_store_uri) = self.client_listen_uri.clone() { - let mut response = MultiViewStoreQueryResponse::new(); - for query in request.queries { - let result = self.query_impl(query); - if let Ok(attested_message) = result { - response.set_query_response(attested_message); - return send_result(ctx, sink, Ok(response), logger); - } - } - - let decryption_error = response.mut_decryption_error(); - decryption_error.set_fog_view_store_uri(fog_view_store_uri.url().to_string()); - decryption_error.set_error_message( - "Could not decrypt a query embedded in the MultiViewStoreQuery".to_string(), - ); + let response = self.process_queries(fog_view_store_uri, request.queries.into_vec()); send_result(ctx, sink, Ok(response), logger) } else { let rpc_permissions_error = rpc_permissions_error( diff --git a/fog/view/server/src/lib.rs b/fog/view/server/src/lib.rs index 42a029f277..c202dce361 100644 --- a/fog/view/server/src/lib.rs +++ b/fog/view/server/src/lib.rs @@ -8,6 +8,7 @@ pub mod fog_view_router_server; pub mod fog_view_router_service; pub mod fog_view_service; pub mod server; +pub mod sharding_strategy; mod block_tracker; mod counters; diff --git a/fog/view/server/src/router_request_handler.rs b/fog/view/server/src/router_request_handler.rs index 823d3bb6bb..dcb5fbb662 100644 --- a/fog/view/server/src/router_request_handler.rs +++ b/fog/view/server/src/router_request_handler.rs @@ -7,6 +7,7 @@ use crate::{ use futures::{future::try_join_all, SinkExt, TryStreamExt}; use grpcio::{ChannelBuilder, DuplexSink, RequestStream, RpcStatus, WriteFlags}; use mc_attest_api::attest; +use mc_attest_enclave_api::{ClientSession, EnclaveMessage}; use mc_common::{logger::Logger, ResponderId}; use mc_fog_api::{ view::{ @@ -18,7 +19,8 @@ use mc_fog_api::{ use mc_fog_uri::FogViewStoreUri; use mc_fog_view_enclave_api::ViewEnclaveProxy; use mc_util_grpc::{rpc_invalid_arg_error, ConnectionUriGrpcioChannel}; -use std::{str::FromStr, sync::Arc}; +use mc_util_uri::ConnectionUri; +use std::{collections::BTreeMap, sync::Arc}; const RETRY_COUNT: usize = 3; @@ -103,12 +105,21 @@ async fn handle_query_request( where E: ViewEnclaveProxy, { - let mut query_responses: Vec = Vec::with_capacity(shard_clients.len()); + let mut query_responses: BTreeMap> = BTreeMap::new(); let mut shard_clients = shard_clients.clone(); + let sealed_query = enclave + .decrypt_and_seal_query(query.into()) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: internal encryption error", + err.into(), + logger.clone(), + ) + })?; // TODO: use retry crate? for _ in 0..RETRY_COUNT { let multi_view_store_query_request = enclave - .create_multi_view_store_query_data(query.clone().into()) + .create_multi_view_store_query_data(sealed_query.clone()) .map_err(|err| { router_server_err_to_rpc_status( "Query: internal encryption error", @@ -128,7 +139,7 @@ where ) })?; - let mut processed_shard_response_data = shard_responses_processor::process_shard_responses( + let processed_shard_response_data = shard_responses_processor::process_shard_responses( clients_and_responses, ) .map_err(|err| { @@ -139,7 +150,13 @@ where ) })?; - query_responses.append(&mut processed_shard_response_data.new_query_responses); + for (store_responder_id, new_query_response) in processed_shard_response_data + .new_query_responses + .into_iter() + { + query_responses.insert(store_responder_id, new_query_response.into()); + } + shard_clients = processed_shard_response_data.shard_clients_for_retry; if shard_clients.is_empty() { break; @@ -153,9 +170,18 @@ where .await?; } - // TODO: Collate the query_responses into one response for the client. Make an - // enclave method for this. - let response = FogViewRouterResponse::new(); + let query_response = enclave + .collate_shard_query_responses(sealed_query, query_responses) + .map_err(|err| { + router_server_err_to_rpc_status( + "Query: shard response collation", + RouterServerError::Enclave(err), + logger.clone(), + ) + })?; + + let mut response = FogViewRouterResponse::new(); + response.set_query(query_response.into()); Ok(response) } @@ -206,7 +232,7 @@ async fn authenticate_view_store( view_store_url: FogViewStoreUri, logger: Logger, ) -> Result<(), RouterServerError> { - let view_store_id = ResponderId::from_str(&view_store_url.to_string())?; + let view_store_id = view_store_url.responder_id()?; let client_auth_request = enclave.view_store_init(view_store_id.clone())?; let grpc_env = Arc::new( grpcio::EnvBuilder::new() diff --git a/fog/view/server/src/server.rs b/fog/view/server/src/server.rs index 8def9929fe..91b7784672 100644 --- a/fog/view/server/src/server.rs +++ b/fog/view/server/src/server.rs @@ -10,6 +10,7 @@ use crate::{ counters, db_fetcher::DbFetcher, fog_view_service::FogViewService, + sharding_strategy::ShardingStrategy, }; use futures::executor::block_on; use mc_attest_net::RaClient; @@ -41,26 +42,28 @@ use std::{ time::{Duration, Instant}, }; -pub struct ViewServer +pub struct ViewServer where E: ViewEnclaveProxy, RC: RaClient + Send + Sync + 'static, DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, { config: MobileAcctViewConfig, server: grpcio::Server, enclave: E, ra_client: RC, report_cache_thread: Option, - db_poll_thread: DbPollThread, + db_poll_thread: DbPollThread, logger: Logger, } -impl ViewServer +impl ViewServer where E: ViewEnclaveProxy, RC: RaClient + Send + Sync + 'static, DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, { /// Make a new view server instance pub fn new( @@ -69,14 +72,16 @@ where recovery_db: DB, ra_client: RC, time_provider: impl TimeProvider + 'static, + sharding_strategy: SS, logger: Logger, - ) -> ViewServer { + ) -> ViewServer { let readiness_indicator = ReadinessIndicator::default(); let db_poll_thread = DbPollThread::new( enclave.clone(), recovery_db.clone(), readiness_indicator.clone(), + sharding_strategy.clone(), logger.clone(), ); @@ -97,16 +102,6 @@ where Arc::new(AnonymousAuthenticator::default()) }; - let fog_view_service = view_grpc::create_fog_view_api(FogViewService::new( - enclave.clone(), - Arc::new(recovery_db), - db_poll_thread.get_shared_state(), - client_authenticator, - config.client_listen_uri.clone(), - logger.clone(), - )); - log::debug!(logger, "Constructed View GRPC Service"); - // Health check service let health_service = mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) @@ -114,6 +109,16 @@ where let server_builder = match config.client_listen_uri { ClientListenUri::ClientFacing(ref fog_view_uri) => { + let fog_view_service = view_grpc::create_fog_view_api(FogViewService::new( + enclave.clone(), + Arc::new(recovery_db), + db_poll_thread.get_shared_state(), + client_authenticator, + config.client_listen_uri.clone(), + sharding_strategy, + logger.clone(), + )); + log::debug!(logger, "Constructed client-facing View GRPC Service"); // Package service into grpc server log::info!(logger, "Starting View server on {}", fog_view_uri.addr(),); grpcio::ServerBuilder::new(env) @@ -122,6 +127,16 @@ where .bind_using_uri(fog_view_uri, logger.clone()) } ClientListenUri::Store(ref fog_view_store_uri) => { + let fog_view_service = view_grpc::create_fog_view_store_api(FogViewService::new( + enclave.clone(), + Arc::new(recovery_db), + db_poll_thread.get_shared_state(), + client_authenticator, + config.client_listen_uri.clone(), + sharding_strategy, + logger.clone(), + )); + log::debug!(logger, "Constructed View Store GRPC Service"); // Package service into grpc server log::info!( logger, @@ -191,11 +206,12 @@ where } } -impl Drop for ViewServer +impl Drop for ViewServer where E: ViewEnclaveProxy, RC: RaClient + Send + Sync + 'static, DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, { fn drop(&mut self) { self.stop(); @@ -217,13 +233,17 @@ pub struct DbPollSharedState { /// The cumulative txo count of the last known block. pub last_known_block_cumulative_txo_count: u64, + + /// The number of blocks that have been processed. + pub processed_block_count: u64, } /// A thread that periodically pushes new tx data from db to enclave -struct DbPollThread +struct DbPollThread where E: ViewEnclaveProxy, DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, { /// Enclave. enclave: E, @@ -243,6 +263,9 @@ where /// Readiness indicator. readiness_indicator: ReadinessIndicator, + /// Sharding strategy, + sharding_strategy: SS, + /// Logger. logger: Logger, } @@ -250,10 +273,11 @@ where /// How long to wait between polling db const DB_POLL_INTERNAL: Duration = Duration::from_millis(100); -impl DbPollThread +impl DbPollThread where E: ViewEnclaveProxy, DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, { /// Get the shared state. pub fn get_shared_state(&self) -> Arc> { @@ -265,6 +289,7 @@ where enclave: E, db: DB, readiness_indicator: ReadinessIndicator, + sharding_strategy: SS, logger: Logger, ) -> Self { let stop_requested = Arc::new(AtomicBool::new(false)); @@ -277,6 +302,7 @@ where stop_requested, shared_state, readiness_indicator, + sharding_strategy, logger, } } @@ -295,6 +321,7 @@ where let thread_stop_requested = self.stop_requested.clone(); let thread_shared_state = self.shared_state.clone(); let thread_readiness_indicator = self.readiness_indicator.clone(); + let thread_sharding_strategy = self.sharding_strategy.clone(); let thread_logger = self.logger.clone(); self.join_handle = Some( @@ -307,6 +334,7 @@ where thread_stop_requested, thread_shared_state, thread_readiness_indicator, + thread_sharding_strategy, thread_logger, ) }) @@ -330,6 +358,7 @@ where stop_requested: Arc, shared_state: Arc>, readiness_indicator: ReadinessIndicator, + sharding_strategy: SS, logger: Logger, ) { log::debug!(logger, "Db poll thread started"); @@ -340,6 +369,7 @@ where db, shared_state, readiness_indicator, + sharding_strategy, logger.clone(), ); loop { @@ -359,10 +389,11 @@ where } } -impl Drop for DbPollThread +impl Drop for DbPollThread where E: ViewEnclaveProxy, DB: RecoveryDb + Clone + Send + Sync + 'static, + SS: ShardingStrategy + Clone + Send + Sync + 'static, { fn drop(&mut self) { let _ = self.stop(); @@ -416,20 +447,24 @@ where E: ViewEnclaveProxy, DB: RecoveryDb + Clone + Send + Sync + 'static, { - pub fn new( + pub fn new( stop_requested: Arc, enclave: E, db: DB, shared_state: Arc>, readiness_indicator: ReadinessIndicator, + sharding_strategy: SS, logger: Logger, - ) -> Self { + ) -> Self + where + SS: ShardingStrategy + Clone + Send + Sync + 'static, + { Self { stop_requested, enclave, db: db.clone(), shared_state, - db_fetcher: DbFetcher::new(db, readiness_indicator, logger.clone()), + db_fetcher: DbFetcher::new(db, readiness_indicator, sharding_strategy, logger.clone()), enclave_block_tracker: BlockTracker::new(logger.clone()), last_unblocked_at: Instant::now(), logger, @@ -606,6 +641,8 @@ where // Track that this block was processed. self.enclave_block_tracker .block_processed(ingress_key, block_index); + let mut shared_state = self.shared_state.lock().expect("mutex poisoned"); + shared_state.processed_block_count += 1; // Update metrics counters::BLOCKS_ADDED_COUNT.inc(); diff --git a/fog/view/server/src/shard_responses_processor.rs b/fog/view/server/src/shard_responses_processor.rs index b5992bcfe3..e19ac55057 100644 --- a/fog/view/server/src/shard_responses_processor.rs +++ b/fog/view/server/src/shard_responses_processor.rs @@ -2,8 +2,13 @@ use crate::error::RouterServerError; use mc_attest_api::attest; -use mc_fog_api::{view::MultiViewStoreQueryResponse, view_grpc::FogViewStoreApiClient}; +use mc_common::ResponderId; +use mc_fog_api::{ + view::{MultiViewStoreQueryResponse, MultiViewStoreQueryResponseStatus}, + view_grpc::FogViewStoreApiClient, +}; use mc_fog_uri::FogViewStoreUri; +use mc_util_uri::ConnectionUri; use std::{str::FromStr, sync::Arc}; /// The result of processing the MultiViewStoreQueryResponse from each Fog View @@ -19,14 +24,14 @@ pub struct ProcessedShardResponseData { pub view_store_uris_for_authentication: Vec, /// New, successfully processed query responses. - pub new_query_responses: Vec, + pub new_query_responses: Vec<(ResponderId, attest::Message)>, } impl ProcessedShardResponseData { pub fn new( shard_clients_for_retry: Vec>, view_store_uris_for_authentication: Vec, - new_query_responses: Vec, + new_query_responses: Vec<(ResponderId, attest::Message)>, ) -> Self { ProcessedShardResponseData { shard_clients_for_retry, @@ -45,16 +50,23 @@ pub fn process_shard_responses( let mut new_query_responses = Vec::new(); for (shard_client, mut response) in clients_and_responses { - // We did not receive a query_response for this shard.Therefore, we need to: - // (a) retry the query - // (b) authenticate with the Fog View Store that returned the decryption_error - if response.has_decryption_error() { - shard_clients_for_retry.push(shard_client); - let store_uri = - FogViewStoreUri::from_str(&response.get_decryption_error().fog_view_store_uri)?; - view_store_uris_for_authentication.push(store_uri); - } else { - new_query_responses.push(response.take_query_response()); + let store_uri = FogViewStoreUri::from_str(response.get_fog_view_store_uri())?; + match response.get_status() { + MultiViewStoreQueryResponseStatus::SUCCESS => { + let store_responder_id = store_uri.responder_id()?; + new_query_responses.push((store_responder_id, response.take_query_response())); + } + // The shard was unable to produce a query response because the Fog View Store + // it contacted isn't authenticated with the Fog View Router. Therefore + // we need to (a) retry the query (b) authenticate with the Fog View + // Store. + MultiViewStoreQueryResponseStatus::AUTHENTICATION_ERROR => { + shard_clients_for_retry.push(shard_client); + view_store_uris_for_authentication.push(store_uri); + } + // Don't do anything if the Fog View Store isn't ready. It's already authenticated, + // hasn't returned a new query response, and shouldn't be retried yet. + MultiViewStoreQueryResponseStatus::NOT_READY => (), } } @@ -70,35 +82,51 @@ mod tests { use super::*; use grpcio::ChannelBuilder; use mc_common::logger::{test_with_logger, Logger}; + use mc_fog_uri::FogViewStoreScheme; use mc_util_grpc::ConnectionUriGrpcioChannel; + use mc_util_uri::UriScheme; - fn create_successful_mvq_response() -> MultiViewStoreQueryResponse { + fn create_successful_mvq_response(client_index: usize) -> MultiViewStoreQueryResponse { let mut successful_response = MultiViewStoreQueryResponse::new(); let client_auth_request = Vec::new(); successful_response .mut_query_response() .set_data(client_auth_request); + let view_uri_string = format!( + "{}://node{}.test.mobilecoin.com:{}", + FogViewStoreScheme::SCHEME_INSECURE, + client_index, + FogViewStoreScheme::DEFAULT_INSECURE_PORT, + ); + successful_response.set_fog_view_store_uri(view_uri_string); + successful_response.set_status(MultiViewStoreQueryResponseStatus::SUCCESS); successful_response } - fn create_failed_mvq_response(i: usize) -> MultiViewStoreQueryResponse { + fn create_failed_mvq_response( + client_index: usize, + status: MultiViewStoreQueryResponseStatus, + ) -> MultiViewStoreQueryResponse { let mut failed_response = MultiViewStoreQueryResponse::new(); let view_uri_string = format!( - "insecure-fog-view-store://node{}.test.mobilecoin.com:3225", - i + "{}://node{}.test.mobilecoin.com:{}", + FogViewStoreScheme::SCHEME_INSECURE, + client_index, + FogViewStoreScheme::DEFAULT_INSECURE_PORT, ); - failed_response - .mut_decryption_error() - .set_fog_view_store_uri(view_uri_string); + failed_response.set_fog_view_store_uri(view_uri_string); + failed_response.set_status(status); failed_response } fn create_grpc_client(i: usize, logger: Logger) -> Arc { let view_uri_string = format!( - "insecure-fog-view-store://node{}.test.mobilecoin.com:3225", - i + "{}://node{}.test.mobilecoin.com:{}", + FogViewStoreScheme::SCHEME_INSECURE, + i, + FogViewStoreScheme::DEFAULT_INSECURE_PORT, ); let view_uri = FogViewStoreUri::from_str(&view_uri_string).unwrap(); let grpc_env = Arc::new( @@ -116,8 +144,9 @@ mod tests { #[test_with_logger] fn one_successful_response_no_shard_clients(logger: Logger) { - let grpc_client = create_grpc_client(0, logger.clone()); - let successful_mvq_response = create_successful_mvq_response(); + let client_index = 0; + let grpc_client = create_grpc_client(client_index, logger.clone()); + let successful_mvq_response = create_successful_mvq_response(client_index); let clients_and_responses = vec![(grpc_client, successful_mvq_response)]; let result = process_shard_responses(clients_and_responses); @@ -130,8 +159,9 @@ mod tests { #[test_with_logger] fn one_successful_response_no_pending_authentications(logger: Logger) { - let grpc_client = create_grpc_client(0, logger.clone()); - let successful_mvq_response = create_successful_mvq_response(); + let client_index = 0; + let grpc_client = create_grpc_client(client_index, logger.clone()); + let successful_mvq_response = create_successful_mvq_response(client_index); let clients_and_responses = vec![(grpc_client, successful_mvq_response)]; let result = process_shard_responses(clients_and_responses); @@ -144,8 +174,9 @@ mod tests { #[test_with_logger] fn one_successful_response_one_new_query_response(logger: Logger) { - let grpc_client = create_grpc_client(0, logger.clone()); - let successful_mvq_response = create_successful_mvq_response(); + let client_index = 0; + let grpc_client = create_grpc_client(client_index, logger.clone()); + let successful_mvq_response = create_successful_mvq_response(client_index); let clients_and_responses = vec![(grpc_client, successful_mvq_response)]; let result = process_shard_responses(clients_and_responses); @@ -157,10 +188,13 @@ mod tests { } #[test_with_logger] - fn one_failed_response_one_pending_shard_client(logger: Logger) { + fn one_auth_error_response_one_pending_shard_client(logger: Logger) { let client_index = 0; let grpc_client = create_grpc_client(client_index, logger.clone()); - let failed_mvq_response = create_failed_mvq_response(client_index); + let failed_mvq_response = create_failed_mvq_response( + client_index, + MultiViewStoreQueryResponseStatus::AUTHENTICATION_ERROR, + ); let clients_and_responses = vec![(grpc_client, failed_mvq_response)]; let result = process_shard_responses(clients_and_responses); @@ -172,10 +206,13 @@ mod tests { } #[test_with_logger] - fn one_failed_response_one_pending_authentications(logger: Logger) { + fn one_auth_error_response_one_pending_authentications(logger: Logger) { let client_index: usize = 0; let grpc_client = create_grpc_client(client_index, logger.clone()); - let failed_mvq_response = create_failed_mvq_response(client_index); + let failed_mvq_response = create_failed_mvq_response( + client_index, + MultiViewStoreQueryResponseStatus::AUTHENTICATION_ERROR, + ); let clients_and_responses = vec![(grpc_client, failed_mvq_response)]; let result = process_shard_responses(clients_and_responses); @@ -187,10 +224,29 @@ mod tests { } #[test_with_logger] - fn one_failed_response_zero_new_query_responses(logger: Logger) { + fn one_auth_error_response_zero_new_query_responses(logger: Logger) { + let client_index: usize = 0; + let grpc_client = create_grpc_client(client_index, logger.clone()); + let failed_mvq_response = create_failed_mvq_response( + client_index, + MultiViewStoreQueryResponseStatus::AUTHENTICATION_ERROR, + ); + let clients_and_responses = vec![(grpc_client, failed_mvq_response)]; + + let result = process_shard_responses(clients_and_responses); + + assert!(result.is_ok()); + + let new_query_responses = result.unwrap().new_query_responses; + assert!(new_query_responses.is_empty()); + } + + #[test_with_logger] + fn one_not_ready_response_zero_new_query_responses(logger: Logger) { let client_index: usize = 0; let grpc_client = create_grpc_client(client_index, logger.clone()); - let failed_mvq_response = create_failed_mvq_response(client_index); + let failed_mvq_response = + create_failed_mvq_response(client_index, MultiViewStoreQueryResponseStatus::NOT_READY); let clients_and_responses = vec![(grpc_client, failed_mvq_response)]; let result = process_shard_responses(clients_and_responses); @@ -202,20 +258,55 @@ mod tests { } #[test_with_logger] - fn mixed_failed_and_successful_responses_processes_correctly(logger: Logger) { + fn one_not_ready_response_zero_pending_authentications(logger: Logger) { + let client_index: usize = 0; + let grpc_client = create_grpc_client(client_index, logger.clone()); + let failed_mvq_response = + create_failed_mvq_response(client_index, MultiViewStoreQueryResponseStatus::NOT_READY); + let clients_and_responses = vec![(grpc_client, failed_mvq_response)]; + + let result = process_shard_responses(clients_and_responses); + + assert!(result.is_ok()); + + let view_store_uris_for_authentication = result.unwrap().view_store_uris_for_authentication; + assert_eq!(view_store_uris_for_authentication.len(), 0); + } + + #[test_with_logger] + fn one_not_ready_response_zero_pending_shard_clients(logger: Logger) { + let client_index: usize = 0; + let grpc_client = create_grpc_client(client_index, logger.clone()); + let failed_mvq_response = + create_failed_mvq_response(client_index, MultiViewStoreQueryResponseStatus::NOT_READY); + let clients_and_responses = vec![(grpc_client, failed_mvq_response)]; + + let result = process_shard_responses(clients_and_responses); + + assert!(result.is_ok()); + + let shard_clients_for_retry = result.unwrap().shard_clients_for_retry; + assert_eq!(shard_clients_for_retry.len(), 0); + } + + #[test_with_logger] + fn mixed_auth_error_and_successful_responses_processes_correctly(logger: Logger) { const NUMBER_OF_FAILURES: usize = 11; const NUMBER_OF_SUCCESSES: usize = 8; let mut clients_and_responses = Vec::new(); for i in 0..NUMBER_OF_FAILURES { let grpc_client = create_grpc_client(i, logger.clone()); - let failed_mvq_response = create_failed_mvq_response(i); + let failed_mvq_response = create_failed_mvq_response( + i, + MultiViewStoreQueryResponseStatus::AUTHENTICATION_ERROR, + ); clients_and_responses.push((grpc_client, failed_mvq_response)); } for i in 0..NUMBER_OF_SUCCESSES { let client_index = i + NUMBER_OF_FAILURES; let grpc_client = create_grpc_client(client_index, logger.clone()); - let successful_mvq_response = create_successful_mvq_response(); + let successful_mvq_response = create_successful_mvq_response(client_index); clients_and_responses.push((grpc_client, successful_mvq_response)); } diff --git a/fog/view/server/src/sharding_strategy.rs b/fog/view/server/src/sharding_strategy.rs new file mode 100644 index 0000000000..766038dae1 --- /dev/null +++ b/fog/view/server/src/sharding_strategy.rs @@ -0,0 +1,241 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +//! Enables a Fog View Store to know for which blocks to process TxOuts. +//! +//! By determining which TxOuts to process, we are able to "shard" the set of +//! TxOuts across Fog View Store instances. + +use mc_blockchain_types::BlockIndex; +use mc_fog_types::{common::BlockRange, BlockCount}; +use serde::Serialize; +use std::str::FromStr; + +/// Tells a Fog View Store for which blocks it should process TxOuts. +pub trait ShardingStrategy { + /// Returns true if the Fog View Store should process this block. + fn should_process_block(&self, block_index: BlockIndex) -> bool; + + /// Returns true if the Fog View Store is ready to serve TxOuts to the + /// client. + /// + /// Different sharding strategies might be ready to serve TxOuts when + /// different conditions have been met. + fn is_ready_to_serve_tx_outs(&self, processed_block_count: BlockCount) -> bool; +} + +/// Determines whether or not to process a block's TxOuts based on the "epoch" +/// sharding strategy, in which a block is processed IFF it falls within the +/// contiguous range of blocks. +/// +/// In practice, the set of Fog View Shards will contain overlapping +/// [epoch_block_ranges] in order to obfuscate which shard processed the TxOuts. +#[derive(Clone, Serialize)] +pub struct EpochShardingStrategy { + /// If a block falls within this range, then the Fog View Store should + /// process its TxOuts. + epoch_block_range: BlockRange, +} + +impl ShardingStrategy for EpochShardingStrategy { + fn should_process_block(&self, block_index: BlockIndex) -> bool { + self.epoch_block_range.contains(block_index) + } + + fn is_ready_to_serve_tx_outs(&self, processed_block_count: BlockCount) -> bool { + self.have_enough_blocks_been_processed(processed_block_count) + } +} + +impl Default for EpochShardingStrategy { + fn default() -> Self { + Self { + epoch_block_range: BlockRange::new(0, u64::MAX), + } + } +} + +impl EpochShardingStrategy { + #[allow(dead_code)] + pub fn new(epoch_block_range: BlockRange) -> Self { + Self { epoch_block_range } + } + + fn have_enough_blocks_been_processed(&self, processed_block_count: BlockCount) -> bool { + if self.is_first_epoch() { + return true; + } + + let epoch_block_range_length = + self.epoch_block_range.end_block - self.epoch_block_range.start_block; + let minimum_processed_block_count = epoch_block_range_length / 2; + + u64::from(processed_block_count) >= minimum_processed_block_count + } + + fn is_first_epoch(&self) -> bool { + self.epoch_block_range.start_block == 0 + } +} + +impl FromStr for EpochShardingStrategy { + type Err = String; + + fn from_str(s: &str) -> Result { + if let Ok(block_range) = BlockRange::from_str(s) { + return Ok(Self::new(block_range)); + } + + Err("Invalid epoch sharding strategy.".to_string()) + } +} + +#[cfg(test)] +mod epoch_sharding_strategy_tests { + use super::*; + + #[test] + fn should_process_block_block_index_is_before_epoch_start_returns_false() { + const START_BLOCK: BlockIndex = 50; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let should_process_block = epoch_sharding_strategy.should_process_block(START_BLOCK - 1); + + assert!(!should_process_block) + } + + #[test] + fn should_process_block_block_index_is_epoch_start_returns_true() { + const START_BLOCK: BlockIndex = 50; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let should_process_block = epoch_sharding_strategy.should_process_block(START_BLOCK); + + assert!(should_process_block) + } + + #[test] + fn should_process_block_block_index_is_in_epoch_block_range_returns_true() { + const START_BLOCK: BlockIndex = 50; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let included_block_index = ((END_BLOCK_EXCLUSIVE - START_BLOCK) / 2) + START_BLOCK; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let should_process_block = + epoch_sharding_strategy.should_process_block(included_block_index); + + assert!(should_process_block) + } + + #[test] + fn should_process_block_block_index_is_one_before_epoch_end_block_range_returns_true() { + const START_BLOCK: BlockIndex = 50; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + let should_process_block = + epoch_sharding_strategy.should_process_block(END_BLOCK_EXCLUSIVE - 1); + assert!(should_process_block) + } + + #[test] + fn should_process_block_block_index_is_epoch_end_block_range_returns_false() { + const START_BLOCK: BlockIndex = 50; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let should_process_block = + epoch_sharding_strategy.should_process_block(END_BLOCK_EXCLUSIVE); + + assert!(!should_process_block) + } + + #[test] + fn should_process_block_block_index_is_after_epoch_end_block_range_returns_false() { + const START_BLOCK: BlockIndex = 50; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let should_process_block = + epoch_sharding_strategy.should_process_block(END_BLOCK_EXCLUSIVE + 1); + + assert!(!should_process_block) + } + + #[test] + fn is_ready_to_serve_tx_outs_allows_0_in_0_to_100_shard() { + // The first epoch has a start block == 0. + const START_BLOCK: BlockIndex = 0; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let is_ready_to_serve_txouts = epoch_sharding_strategy.is_ready_to_serve_tx_outs(0.into()); + + assert!(is_ready_to_serve_txouts) + } + + #[test] + fn is_ready_to_serve_tx_outs_allows_70_in_0_to_100_shard() { + // The first epoch has a start block == 0. + const START_BLOCK: BlockIndex = 0; + const END_BLOCK_EXCLUSIVE: BlockIndex = 100; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let is_ready_to_serve_txouts = epoch_sharding_strategy.is_ready_to_serve_tx_outs(70.into()); + + assert!(is_ready_to_serve_txouts) + } + + #[test] + fn is_ready_to_serve_tx_outs_not_first_shard_prevents_less_than_minimum() { + const START_BLOCK: BlockIndex = 100; + const END_BLOCK_EXCLUSIVE: BlockIndex = 111; + let epoch_block_range_length = END_BLOCK_EXCLUSIVE - START_BLOCK; + let minimum_processed_block_count = epoch_block_range_length / 2; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let is_ready_to_serve_txouts = epoch_sharding_strategy + .is_ready_to_serve_tx_outs((minimum_processed_block_count - 1).into()); + + assert!(!is_ready_to_serve_txouts) + } + + #[test] + fn is_ready_to_serve_tx_outs_not_first_shard_allows_minimum() { + const START_BLOCK: BlockIndex = 100; + const END_BLOCK_EXCLUSIVE: BlockIndex = 111; + let epoch_block_range_length = END_BLOCK_EXCLUSIVE - START_BLOCK; + let minimum_processed_block_count = epoch_block_range_length / 2; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let is_ready_to_serve_txouts = + epoch_sharding_strategy.is_ready_to_serve_tx_outs(minimum_processed_block_count.into()); + + assert!(is_ready_to_serve_txouts) + } + + #[test] + fn is_ready_to_serve_tx_outs_not_first_shard_allows_over_minimum() { + const START_BLOCK: BlockIndex = 100; + const END_BLOCK_EXCLUSIVE: BlockIndex = 110; + let epoch_block_range_length = END_BLOCK_EXCLUSIVE - START_BLOCK; + let minimum_processed_block_count = epoch_block_range_length / 2; + let epoch_block_range = BlockRange::new(START_BLOCK, END_BLOCK_EXCLUSIVE); + let epoch_sharding_strategy = EpochShardingStrategy::new(epoch_block_range); + + let is_ready_to_serve_txouts = epoch_sharding_strategy + .is_ready_to_serve_tx_outs((minimum_processed_block_count + 1).into()); + + assert!(is_ready_to_serve_txouts) + } +} diff --git a/fog/view/server/tests/smoke_tests.rs b/fog/view/server/tests/smoke_tests.rs index d096f986d8..424064a477 100644 --- a/fog/view/server/tests/smoke_tests.rs +++ b/fog/view/server/tests/smoke_tests.rs @@ -34,8 +34,9 @@ use mc_fog_view_connection::FogViewGrpcClient; use mc_fog_view_enclave::SgxViewEnclave; use mc_fog_view_protocol::FogViewConnection; use mc_fog_view_server::{ - config::{ClientListenUri::ClientFacing, MobileAcctViewConfig as ViewConfig}, + config::{ClientListenUri::ClientFacing, MobileAcctViewConfig as ViewConfig, ShardingStrategy}, server::ViewServer, + sharding_strategy::EpochShardingStrategy, }; use mc_util_from_random::FromRandom; use mc_util_grpc::GrpcRetryConfig; @@ -52,7 +53,7 @@ fn get_test_environment( logger: Logger, ) -> ( SqlRecoveryDbTestContext, - ViewServer, + ViewServer, FogViewGrpcClient, ) { let db_test_context = SqlRecoveryDbTestContext::new(logger.clone()); @@ -71,6 +72,7 @@ fn get_test_environment( ias_api_key: Default::default(), admin_listen_uri: Default::default(), client_auth_token_max_lifetime: Default::default(), + sharding_strategy: ShardingStrategy::Epoch(EpochShardingStrategy::default()), postgres_config: Default::default(), }; @@ -90,6 +92,7 @@ fn get_test_environment( db, ra_client, SystemTimeProvider::default(), + EpochShardingStrategy::default(), logger.clone(), ); server.start();