Problem: no proof that a keypair was generated inside NE (fixes #92)

Solution:
- extended response for keygen in the helper + extra comments/small refactor
- nitro enclave returns an attestation doc for the generated pubkey + used AWS KMS key id
- added a small Python verification script to illustrate how the attestation doc payload can be processed

Dockerfile.nitro

@@ -167,7 +167,7 @@ RUN git clone https://github.com/json-c/json-c.git \
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN
 
 # NSM LIB
-ENV AWS_NE_NSM_API_VER="v0.1.0"
+ENV AWS_NE_NSM_API_VER="34bad95f97f8c83a844e1db8695e91552b1aa9f3"
 RUN git clone "https://github.com/aws/aws-nitro-enclaves-nsm-api" \
     && cd aws-nitro-enclaves-nsm-api \
     && git reset --hard $AWS_NE_NSM_API_VER \

NOTICE

@@ -16,4 +16,14 @@ This project contains portions of code derived from the following libraries:
 * tokio-rs/tracing
  * Copyright: Copyright (c) 2019 Tokio Contributors
  * License: MIT License
- * Repository: https://github.com/tokio-rs/tracing
+ * Repository: https://github.com/tokio-rs/tracing
+
+* AWS Nitro Enclaves Python demo
+ * Copyright: Copyright 2020 Richard Fan
+ * License: Apache License 2.0
+ * Repository: https://github.com/richardfan1126/nitro-enclave-python-demo
+
+* bech32
+ * Copyright: Copyright (c) 2017 Pieter Wuille
+ * License: MIT License
+ * Repository: https://github.com/fiatjaf/bech32

providers/nitro/nitro-enclave/Cargo.toml

@@ -9,9 +9,13 @@ anomaly = "0.2"
 aws-ne-sys = "0.4"
 ed25519-dalek = "1"
 nix = "0.21"
+nsm-io = { git = "https://github.com/aws/aws-nitro-enclaves-nsm-api", rev="34bad95f97f8c83a844e1db8695e91552b1aa9f3" }
+nsm-driver = { git = "https://github.com/aws/aws-nitro-enclaves-nsm-api", rev="34bad95f97f8c83a844e1db8695e91552b1aa9f3" }
 rand_core = { version = "0.6", default-features = false, features = ["getrandom"]}
+serde_bytes = "0.11"
 serde_json = "1"
 subtle = "2"
+subtle-encoding = "0.5"
 tendermint = { version = "0.19" }
 tendermint-p2p = { version = "0.19" }
 tmkms-light = { path = "../../.." }

providers/nitro/nitro-enclave/src/nitro.rs

@@ -4,7 +4,10 @@ mod state;
 use anomaly::format_err;
 use ed25519_dalek as ed25519;
 use ed25519_dalek::Keypair;
+use nsm_driver::{nsm_exit, nsm_init, nsm_process_request};
+use nsm_io::{Request, Response};
 use rand_core::OsRng;
+use serde_bytes::ByteBuf;
 use std::io;
 use std::os::unix::io::AsRawFd;
 use std::thread;
@@ -20,10 +23,12 @@ use tmkms_light::error::{
     ErrorKind::{AccessError, InvalidKey, IoError, ParseError},
 };
 use tmkms_light::utils::{read_u16_payload, write_u16_payload};
-use tmkms_nitro_helper::{NitroConfig, NitroRequest, NitroResponse, VSOCK_HOST_CID};
+use tmkms_nitro_helper::{
+    NitroConfig, NitroKeygenResponse, NitroRequest, NitroResponse, VSOCK_HOST_CID,
+};
 use tracing::{error, info, trace, warn};
 use vsock::{SockAddr, VsockStream};
-use zeroize::Zeroizing;
+use zeroize::{Zeroize, Zeroizing};
 
 fn get_secret_connection(
     vsock_port: u32,
@@ -100,11 +105,12 @@ pub fn get_connection(
 
 /// a simple req-rep handling loop
 pub fn entry(mut stream: VsockStream) -> Result<(), Error> {
+    let nsm_fd = nsm_init();
     let json_raw = read_u16_payload(&mut stream)
         .map_err(|_e| format_err!(IoError, "failed to read config"))?;
     let request: Result<NitroRequest, _> = serde_json::from_slice(&json_raw);
     match request {
-        Ok(NitroRequest::Config(config)) => {
+        Ok(NitroRequest::Start(config)) => {
             let key_bytes = Zeroizing::new(
                 aws_ne_sys::kms_decrypt(
                     config.aws_region.as_bytes(),
@@ -167,22 +173,51 @@ pub fn entry(mut stream: VsockStream) -> Result<(), Error> {
         }
         Ok(NitroRequest::Keygen(keygen_config)) => {
             let mut csprng = OsRng {};
-            let keypair = Keypair::generate(&mut csprng);
+            let mut keypair = Keypair::generate(&mut csprng);
             let public = keypair.public;
-            let response = match aws_ne_sys::kms_encrypt(
+            let pubkeyb64 = String::from_utf8(subtle_encoding::base64::encode(&public))
+                .map_err(|e| format_err!(IoError, "base64 encoding error: {:?}", e))?;
+            let keyidb64 =
+                String::from_utf8(subtle_encoding::base64::encode(&keygen_config.kms_key_id))
+                    .map_err(|e| format_err!(IoError, "base64 encoding error: {:?}", e))?;
+
+            let claim = format!(
+                "{{\"pubkey\":\"{}\",\"key_id\":\"{}\"}}",
+                pubkeyb64, keyidb64
+            );
+            let user_data = Some(ByteBuf::from(claim));
+            let response: NitroResponse = match aws_ne_sys::kms_encrypt(
                 keygen_config.aws_region.as_bytes(),
                 keygen_config.credentials.aws_key_id.as_bytes(),
                 keygen_config.credentials.aws_secret_key.as_bytes(),
                 keygen_config.credentials.aws_session_token.as_bytes(),
                 keygen_config.kms_key_id.as_bytes(),
                 keypair.secret.as_bytes(),
             ) {
-                Ok(cipher_privkey) => {
-                    NitroResponse::CipherKeypair((cipher_privkey, public.as_bytes().to_vec()))
+                Ok(encrypted_secret) => {
+                    let req = Request::Attestation {
+                        user_data,
+                        // as this is one-off attestation on generation,
+                        // no need here (this may useful in other scenarios)
+                        nonce: None,
+                        // this field is meant for encryptions (e.g. when AWS KMS
+                        // sends a response to the enclave),
+                        // so it's used in `aws_ne_sys`, but not here
+                        public_key: None,
+                    };
+                    let att = nsm_process_request(nsm_fd, req);
+                    match att {
+                        Response::Attestation { document } => Ok(NitroKeygenResponse {
+                            encrypted_secret,
+                            public_key: public.as_bytes().to_vec(),
+                            attestation_doc: document,
+                        }),
+                        _ => Err("failed to obtain an attestation document".to_owned()),
+                    }
                 }
-                Err(e) => NitroResponse::Error(format!("{:?}", e)),
+                Err(e) => Err(format!("{:?}", e)),
             };
-
+            keypair.secret.zeroize();
             let json = serde_json::to_string(&response)
                 .map_err(|e| format_err!(ParseError, "serde keygen response error: {:?}", e))?;
             write_u16_payload(&mut stream, json.as_bytes())
@@ -192,6 +227,7 @@ pub fn entry(mut stream: VsockStream) -> Result<(), Error> {
             error!("config error: {}", e);
         }
     }
+    nsm_exit(nsm_fd);
 
     Ok(())
 }

providers/nitro/nitro-helper/src/command.rs

@@ -54,7 +54,7 @@ pub fn init(
             .ok_or_else(|| "cannot create a dir in a root directory".to_owned())?,
     )
     .map_err(|e| format!("failed to create dirs for state storage: {:?}", e))?;
-    let pubkey = generate_key(
+    let (pubkey, attestation_doc) = generate_key(
         cid,
         port,
         config.sealed_consensus_key_path,
@@ -64,6 +64,10 @@ pub fn init(
     )
     .map_err(|e| format!("failed to generate a key: {:?}", e))?;
     print_pubkey(bech32_prefix, pubkey_display, pubkey);
+    let encoded_attdoc = String::from_utf8(subtle_encoding::base64::encode(&attestation_doc))
+        .map_err(|e| format!("enconding attestation doc: {:?}", e))?;
+    println!("Nitro Enclave attestation:\n{}", &encoded_attdoc);
+
     if let Some(id_path) = config.sealed_id_key_path {
         generate_key(
             cid,
@@ -151,7 +155,7 @@ pub fn start(config_path: Option<PathBuf>, cid: Option<u32>) -> Result<(), Strin
                 e
             )
         })?;
-        let request = NitroRequest::Config(enclave_config);
+        let request = NitroRequest::Start(enclave_config);
         let config_raw = serde_json::to_vec(&request)
             .map_err(|e| format!("failed to serialize the config: {:?}", e))?;
         write_u16_payload(&mut socket, &config_raw)

providers/nitro/nitro-helper/src/key_utils.rs

@@ -1,10 +1,9 @@
 use crate::shared::AwsCredentials;
-use crate::shared::{NitroKeygenConfig, NitroRequest};
+use crate::shared::{NitroKeygenConfig, NitroKeygenResponse, NitroRequest, NitroResponse};
 
 use ed25519_dalek::PublicKey;
 use std::{fs::OpenOptions, io::Write, os::unix::fs::OpenOptionsExt, path::Path};
 use tmkms_light::utils::{read_u16_payload, write_u16_payload};
-use tmkms_nitro_helper::NitroResponse;
 use vsock::SockAddr;
 
 // TODO: use aws-rust-sdk after the issue fixed
@@ -99,15 +98,17 @@ pub(crate) mod credential {
     }
 }
 
-/// Generates key and encrypts with AWS KMS at the given path
+/// Generates a keypair and encrypts with AWS KMS at the given path
+/// and returns the public key with attestation doc for it and
+/// the used AWS KMS key id
 pub fn generate_key(
     cid: u32,
     port: u32,
     path: impl AsRef<Path>,
     region: &str,
     credentials: AwsCredentials,
     kms_key_id: String,
-) -> Result<PublicKey, String> {
+) -> Result<(PublicKey, Vec<u8>), String> {
     let keygen_request = NitroKeygenConfig {
         credentials,
         kms_key_id,
@@ -132,17 +133,18 @@ pub fn generate_key(
     let response: NitroResponse = serde_json::from_slice(&json_raw)
         .map_err(|e| format!("failed to get keygen response from enclave: {:?}", e))?;
 
-    let (cipher_privkey, pubkey) = match response {
-        NitroResponse::Error(e) => Err(e),
-        NitroResponse::CipherKeypair((cipher, public)) => Ok((cipher, public)),
-    }?;
+    let resp: NitroKeygenResponse = response?;
     OpenOptions::new()
         .create(true)
         .write(true)
         .truncate(true)
         .mode(0o600)
         .open(path.as_ref())
-        .and_then(|mut file| file.write_all(&cipher_privkey))
+        .and_then(|mut file| file.write_all(&resp.encrypted_secret))
         .map_err(|e| format!("couldn't write `{}`: {}", path.as_ref().display(), e))?;
-    PublicKey::from_bytes(&pubkey).map_err(|e| format!("Invalid pubkey key: {:?}", e))
+    Ok((
+        PublicKey::from_bytes(&resp.public_key)
+            .map_err(|e| format!("Invalid pubkey key: {:?}", e))?,
+        resp.attestation_doc,
+    ))
 }

providers/nitro/nitro-helper/src/shared.rs

@@ -28,6 +28,7 @@ pub struct NitroConfig {
     pub aws_region: String,
 }
 
+/// configuration sent during key generation
 #[derive(Debug, Serialize, Deserialize)]
 pub struct NitroKeygenConfig {
     /// AWS credentials -- if not set, they'll be obtained from IAM
@@ -38,19 +39,29 @@ pub struct NitroKeygenConfig {
     pub aws_region: String,
 }
 
+/// types of initial requests sent to NE
 #[derive(Debug, Serialize, Deserialize)]
 pub enum NitroRequest {
+    /// generate a key
     Keygen(NitroKeygenConfig),
-    Config(NitroConfig),
+    /// start up TMKMS processing
+    Start(NitroConfig),
 }
 
+/// response from key generation
 #[derive(Debug, Serialize, Deserialize)]
-pub enum NitroResponse {
-    // (cipher_privkey, public_key)
-    CipherKeypair((Vec<u8>, Vec<u8>)),
-    Error(String),
+pub struct NitroKeygenResponse {
+    /// payload returned from AWS KMS
+    pub encrypted_secret: Vec<u8>,
+    /// public key for consensus or P2P
+    pub public_key: Vec<u8>,
+    /// attestation payload (COSE_Sign1) for the public key + encryption key id
+    pub attestation_doc: Vec<u8>,
 }
 
+/// response from the enclave
+pub type NitroResponse = Result<NitroKeygenResponse, String>;
+
 /// Credentials, generally obtained from parent instance IAM
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]

script/tmkms-nitro/default.nix

@@ -0,0 +1,33 @@
+with import <nixpkgs> {};
+
+( let
+    cose = pkgs.python39Packages.buildPythonPackage rec {
+      pname = "cose";
+      version = "0.9.dev7";
+
+      src = pkgs.python39Packages.fetchPypi{
+        inherit pname version;  
+        sha256 = "d82cb1ebcdc5c759c1307f7302c5e6cb327d4195c03c31cb5fbdf6851a74d7ea";
+      };
+     doCheck = false;
+    preConfigure = ''
+    touch requirements.txt
+  '';
+    };
+    attr = pkgs.python39Packages.buildPythonPackage rec {
+      pname = "attrs";
+      version = "21.2.0";
+
+      src = pkgs.python39Packages.fetchPypi{
+        inherit pname version;  
+        sha256 = "ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb";
+      };
+     doCheck = false;
+
+    };
+
+  in pkgs.python39.buildEnv.override rec {
+
+    extraLibs = [ pkgs.python39Packages.pycryptodome pkgs.python39Packages.cbor2 cose attr pkgs.python39Packages.cryptography pkgs.python39Packages.ecdsa pkgs.python39Packages.pyopenssl ];
+}
+).env