[k8s] Add all root CA certs to TLS connections (Azure#3616)

* Add all root CA certs to TLS connections

* Use openssl stack_from_pem to parse trusted roots

edgelet/edgelet-http/src/authentication.rs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-#![deny(rust_2018_idioms, warnings)]
+#![deny(warnings)]
 #![deny(clippy::all, clippy::pedantic)]
 #![allow(clippy::module_name_repetitions, clippy::use_self)]
 

edgelet/iotedge-proxy/Cargo.toml

@@ -16,6 +16,7 @@ hyper = "0.12"
 hyper-tls = "0.3"
 log = "0.4"
 native-tls = "0.2"
+openssl = "0.10"
 serde = "1.0"
 serde_json = "1.0"
 serde_derive = "1.0"
@@ -26,4 +27,3 @@ url_serde = "0.2"
 
 [dev-dependencies]
 tempfile = "3"
-openssl = "0.10"

edgelet/iotedge-proxy/src/proxy/config.rs

@@ -4,6 +4,7 @@ use std::fs;
 
 use failure::ResultExt;
 use native_tls::{Certificate, TlsConnector};
+use openssl::x509::X509;
 use url::Url;
 
 use crate::{Error, ErrorKind, InitializeErrorReason, ServiceSettings};
@@ -51,10 +52,28 @@ pub fn get_config(settings: &ServiceSettings) -> Result<Config<ValueToken>, Erro
  InitializeErrorReason::ClientConfigReadFile(path.display().to_string()),
  ))?;
 
- let cert = Certificate::from_pem(file.as_bytes())
- .context(ErrorKind::Initialize(InitializeErrorReason::ClientConfig))?;
+ if file.is_empty() {
+ return Err(Error::from(ErrorKind::Initialize(
+ InitializeErrorReason::ClientConfig,
+ )));
+ }
 
- tls.add_root_certificate(cert);
+ let certs = X509::stack_from_pem(file.as_bytes())
+ .context(ErrorKind::Initialize(InitializeErrorReason::ClientConfig))?;
+ if certs.is_empty() {
+ // Expect this has at least one usable certificate
+ return Err(Error::from(ErrorKind::Initialize(
+ InitializeErrorReason::ClientConfig,
+ )));
+ }
+ for cert in certs {
+ let der = cert
+ .to_der()
+ .context(ErrorKind::Initialize(InitializeErrorReason::ClientConfig))?;
+ let cert = Certificate::from_der(&der)
+ .context(ErrorKind::Initialize(InitializeErrorReason::ClientConfig))?;
+ tls.add_root_certificate(cert);
+ }
  }
 
  Ok(Config::new(
@@ -169,7 +188,33 @@ mod tests {
  }
 
  #[test]
- fn it_fails_to_load_config_if_cert_is_invalid() {
+ fn it_fails_to_load_config_if_cert_is_empty() {
+ let dir = TempDir::new().unwrap();
+
+ let token = dir.path().join("token");
+ fs::write(&token, "token").unwrap();
+
+ let cert = dir.path().join("cert.pem");
+ fs::write(&cert, "").unwrap();
+
+ let settings = ServiceSettings::new(
+ "management".to_owned(),
+ Url::parse("http://localhost:3000").unwrap(),
+ Url::parse("https://iotedged:30000").unwrap(),
+ Some(&cert),
+ &token,
+ );
+
+ let err = get_config(&settings).err().unwrap();
+
+ assert_eq!(
+ err.kind(),
+ &ErrorKind::Initialize(InitializeErrorReason::ClientConfig)
+ );
+ }
+
+ #[test]
+ fn it_fails_to_load_config_if_no_cert_delimiter() {
  let dir = TempDir::new().unwrap();
 
  let token = dir.path().join("token");
@@ -193,4 +238,30 @@ mod tests {
  &ErrorKind::Initialize(InitializeErrorReason::ClientConfig)
  );
  }
+
+ #[test]
+ fn it_fails_to_load_config_if_cert_is_invalid() {
+ let dir = TempDir::new().unwrap();
+
+ let token = dir.path().join("token");
+ fs::write(&token, "token").unwrap();
+
+ let cert = dir.path().join("cert.pem");
+ fs::write(&cert, "cert-----END CERTIFICATE-----").unwrap();
+
+ let settings = ServiceSettings::new(
+ "management".to_owned(),
+ Url::parse("http://localhost:3000").unwrap(),
+ Url::parse("https://iotedged:30000").unwrap(),
+ Some(&cert),
+ &token,
+ );
+
+ let err = get_config(&settings).err().unwrap();
+
+ assert_eq!(
+ err.kind(),
+ &ErrorKind::Initialize(InitializeErrorReason::ClientConfig)
+ );
+ }
 }

edgelet/kube-client/src/config.rs

@@ -98,7 +98,7 @@ impl<T: TokenSource> Config<T> {
  })?;
 
  // add the root ca cert to the TLS settings
- let root_ca = Certificate::from_pem(&file_or_data_bytes(
+ let root_ca = get_all_certs(file_or_data_bytes(
  cluster.certificate_authority(),
  cluster.certificate_authority_data(),
  )?)
@@ -123,14 +123,20 @@ impl<T: TokenSource> Config<T> {
  &file_or_data_bytes(user.client_key(), user.client_key_data())?,
  )?;
 
- TlsConnector::builder()
- .add_root_certificate(root_ca)
+ let mut builder = TlsConnector::builder();
+ for cert in root_ca {
+ builder.add_root_certificate(cert);
+ }
+ builder
  .identity(identity)
  .build()
  .context(ErrorKind::KubeConfig(KubeConfigErrorReason::Tls))?
  } else {
- TlsConnector::builder()
- .add_root_certificate(root_ca)
+ let mut builder = TlsConnector::builder();
+ for cert in root_ca {
+ builder.add_root_certificate(cert);
+ }
+ builder
  .build()
  .context(ErrorKind::KubeConfig(KubeConfigErrorReason::Tls))?
  };
@@ -228,14 +234,19 @@ fn get_token_and_tls_connector() -> Result<(ValueToken, TlsConnector)> {
  let cert = fs::read(ROOT_CA_FILE).context(ErrorKind::KubeConfig(
  KubeConfigErrorReason::LoadCertificate,
  ))?;
- let root_ca = Certificate::from_pem(&cert).context(ErrorKind::KubeConfig(
+ let root_ca = get_all_certs(cert).context(ErrorKind::KubeConfig(
  KubeConfigErrorReason::LoadCertificate,
  ))?;
 
- let tls_connector = TlsConnector::builder()
- .add_root_certificate(root_ca)
- .build()
- .context(ErrorKind::KubeConfig(KubeConfigErrorReason::Tls))?;
+ let tls_connector = {
+ let mut builder = TlsConnector::builder();
+ for cert in root_ca {
+ builder.add_root_certificate(cert);
+ }
+ builder
+ .build()
+ .context(ErrorKind::KubeConfig(KubeConfigErrorReason::Tls))?
+ };
 
  Ok((ValueToken(Some(token)), tls_connector))
 }
@@ -269,6 +280,30 @@ fn identity_from_cert_key(user_name: &str, cert: &[u8], key: &[u8]) -> Result<Id
  Ok(identity)
 }
 
+fn get_all_certs(raw_certs: Vec<u8>) -> Result<Vec<Certificate>> {
+ let certs = X509::stack_from_pem(&raw_certs).context(ErrorKind::KubeConfig(
+ KubeConfigErrorReason::LoadCertificate,
+ ))?;
+ if certs.is_empty() {
+ // Expect this has at least one usable certificate
+ return Err(Error::from(ErrorKind::KubeConfig(
+ KubeConfigErrorReason::LoadCertificate,
+ )));
+ }
+ certs
+ .into_iter()
+ .map(|cert| {
+ let der = cert.to_der().context(ErrorKind::KubeConfig(
+ KubeConfigErrorReason::LoadCertificate,
+ ))?;
+ let cert = Certificate::from_der(&der).context(ErrorKind::KubeConfig(
+ KubeConfigErrorReason::LoadCertificate,
+ ))?;
+ Ok(cert)
+ })
+ .collect()
+}
+
 fn file_or_data_bytes(path: Option<&str>, data: Option<&str>) -> Result<Vec<u8>> {
  // the "data" always overrides the file path
  match data {
@@ -306,6 +341,51 @@ mod tests {
  use std::io::Write;
  use tempdir::TempDir;
 
+ #[test]
+ fn get_all_certs_with_no_good_certs() {
+ let empty = String::new();
+ let not_utf8 = vec![0, 159, 146, 150];
+ let not_a_cert = String::from("not a cert");
+ let bad_cert = String::from("not correct-----END CERTIFICATE-----");
+
+ let empty_result = get_all_certs(empty.into_bytes());
+ let not_utf8_result = get_all_certs(not_utf8);
+ let not_a_cert_result = get_all_certs(not_a_cert.into_bytes());
+ let bad_cert_result = get_all_certs(bad_cert.into_bytes());
+
+ assert!(empty_result.is_err());
+ assert!(not_utf8_result.is_err());
+ assert!(not_a_cert_result.is_err());
+ assert!(bad_cert_result.is_err());
+ }
+
+ #[test]
+ fn get_all_certs_get_single_cert_gets_one_cert() {
+ let one_cert = CertGenerator::default().generate().unwrap();
+
+ let one_cert_result = get_all_certs(one_cert).unwrap();
+
+ assert_eq!(one_cert_result.len(), 1);
+ }
+
+ #[test]
+ fn get_all_certs_multiple_certs_gets_all_certs() {
+ let cert1 = CertGenerator::default().generate().unwrap();
+ let cert1 = std::str::from_utf8(&cert1).unwrap();
+ let cert2 = CertGenerator::default().generate().unwrap();
+ let cert2 = std::str::from_utf8(&cert2).unwrap();
+ let cert3 = CertGenerator::default().generate().unwrap();
+ let cert3 = std::str::from_utf8(&cert3).unwrap();
+ let multiple_certs1 = format!("{}\n{}\nnot a cert", cert1, cert2);
+ let multiple_certs2 = format!("{}\n{}\n{}", cert1, cert2, cert3);
+
+ let cert1_result = get_all_certs(multiple_certs1.into_bytes()).unwrap();
+ let cert2_result = get_all_certs(multiple_certs2.into_bytes()).unwrap();
+
+ assert_eq!(cert1_result.len(), 2);
+ assert_eq!(cert2_result.len(), 3);
+ }
+
  #[test]
  fn test_get_host() {
  let env_key = "KUBERNETES_SERVICE_HOST";

edgelet/kube-client/src/error.rs

@@ -97,6 +97,8 @@ pub enum KubeConfigErrorReason {
  MissingUser,
  MissingData,
  Base64Decode,
+ StringConvert,
+ NoRootData,
  UrlParse(String),
  Tls,
  MissingEnvVar(String),
@@ -120,6 +122,8 @@ impl Display for KubeConfigErrorReason {
  Self::MissingUser => write!(f, "Missing user configuration in .kube/config file."),
  Self::MissingData => write!(f, "Both file and data missing."),
  Self::Base64Decode => write!(f, "Base64 decode error."),
+ Self::StringConvert => write!(f, "Could not convert cert data to string."),
+ Self::NoRootData => write!(f, "Root CA certficate data empty"),
  Self::UrlParse(x) => write!(f, "Unable to parse valid URL from: {}.", x),
  Self::Tls => write!(f, "Could not create TLS connector."),
  Self::MissingEnvVar(x) => write!(f, "Missing ENV: {}.", x),

edgelet/kube-client/src/lib.rs

@@ -21,3 +21,109 @@ pub mod kube;
 pub use self::client::{Client, HttpClient};
 pub use self::config::{get_config, Config, TokenSource, ValueToken};
 pub use self::error::{Error, ErrorKind, RequestType};
+
+#[cfg(test)]
+#[allow(dead_code)]
+mod tls {
+
+ use failure::Fail;
+ use openssl::asn1::Asn1Time;
+ use openssl::error::ErrorStack;
+ use openssl::hash::MessageDigest;
+ use openssl::nid::Nid;
+ use openssl::pkey::PKey;
+ use openssl::rsa::Rsa;
+ use openssl::x509::extension::{
+ AuthorityKeyIdentifier, BasicConstraints, ExtendedKeyUsage, KeyUsage, SubjectKeyIdentifier,
+ };
+ use openssl::x509::{X509Name, X509};
+ use std::fmt::{Display, Formatter};
+ use std::io::Error as IoError;
+
+ #[derive(Debug, Fail)]
+ pub enum CertGeneratorError {
+ ErrorStack(ErrorStack),
+ Io(IoError),
+ }
+
+ impl Display for CertGeneratorError {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ match self {
+ CertGeneratorError::ErrorStack(err) => write!(f, "{}", err),
+ CertGeneratorError::Io(err) => write!(f, "{}", err),
+ }
+ }
+ }
+
+ impl From<ErrorStack> for CertGeneratorError {
+ fn from(err: ErrorStack) -> Self {
+ CertGeneratorError::ErrorStack(err)
+ }
+ }
+
+ impl From<IoError> for CertGeneratorError {
+ fn from(err: IoError) -> Self {
+ CertGeneratorError::Io(err)
+ }
+ }
+
+ #[derive(Default)]
+ pub struct CertGenerator {
+ common_name: Option<String>,
+ }
+
+ impl CertGenerator {
+ pub fn common_name(&mut self, name: String) -> &Self {
+ self.common_name = Some(name);
+ self
+ }
+
+ pub fn generate(&self) -> Result<Vec<u8>, CertGeneratorError> {
+ let rsa = Rsa::generate(2048)?;
+ let pkey = PKey::from_rsa(rsa)?;
+
+ let mut name = X509Name::builder()?;
+ name.append_entry_by_nid(
+ Nid::COMMONNAME,
+ self.common_name
+ .as_ref()
+ .unwrap_or(&"localhost".to_string()),
+ )?;
+ let name = name.build();
+
+ let mut builder = X509::builder()?;
+ builder.set_version(2)?;
+ builder.set_subject_name(&name)?;
+ builder.set_issuer_name(&name)?;
+ builder.set_not_before(Asn1Time::days_from_now(0)?.as_ref())?;
+ builder.set_not_after(Asn1Time::days_from_now(365)?.as_ref())?;
+ builder.set_pubkey(&pkey)?;
+
+ let basic_constraints = BasicConstraints::new().critical().ca().build()?;
+ builder.append_extension(basic_constraints)?;
+ let key_usage = KeyUsage::new()
+ .digital_signature()
+ .key_encipherment()
+ .build()?;
+ builder.append_extension(key_usage)?;
+ let ext_key_usage = ExtendedKeyUsage::new()
+ .client_auth()
+ .server_auth()
+ .build()?;
+ builder.append_extension(ext_key_usage)?;
+ let subject_key_identifier =
+ SubjectKeyIdentifier::new().build(&builder.x509v3_context(None, None))?;
+ builder.append_extension(subject_key_identifier)?;
+ let authority_key_identifier = AuthorityKeyIdentifier::new()
+ .keyid(true)
+ .build(&builder.x509v3_context(None, None))?;
+ builder.append_extension(authority_key_identifier)?;
+
+ builder.sign(&pkey, MessageDigest::sha256())?;
+
+ let x509 = builder.build().to_pem()?;
+
+ Ok(x509)
+ }
+ }
+}