encryption: change to use openssl EVP API (facebook#156)

Summary:
Instead of using openssl's raw `AES_encrypt` and `AES_decrypt` API, which is a low level call to encrypt or decrypt exact one block (16 bytes), we change to use the `EVP_*` API. The former is deprecated, and will use the default C implementation without AES-NI support. Also the EVP API is capable of handing CTR mode on its own.

Test Plan:
will add tests

Signed-off-by: Yi Wu <yiwu@pingcap.com>
Signed-off-by: tabokie <xy.tao@outlook.com>
(cherry picked from commit 63399df)

CMakeLists.txt

@@ -1368,6 +1368,7 @@ if(WITH_TESTS)
  db/write_batch_test.cc
  db/write_callback_test.cc
  db/write_controller_test.cc
+ encryption/encryption_test.cc
  env/env_test.cc
  env/io_posix_test.cc
  env/mock_env_test.cc

Makefile

@@ -703,6 +703,7 @@ TESTS_PLATFORM_DEPENDENT := \
  crc32c_test \
  coding_test \
  inlineskiplist_test \
+ encryption_test \
  env_basic_test \
  env_test \
  env_logger_test \
@@ -1979,6 +1980,8 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.
  $(AM_LINK)
 
 wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY)
+
+encryption_test: encryption/encryption_test.o $(LIBOBJECTS) $(TESTHARNESS)
  $(AM_LINK)
 
 #-------------------------------------------------

encryption/encryption.cc

@@ -1,64 +1,222 @@
-// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
-// This source code is licensed under both the GPLv2 (found in the
-// COPYING file in the root directory) and Apache 2.0 License
-// (found in the LICENSE.Apache file in the root directory).
+// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
 
 #ifndef ROCKSDB_LITE
 #ifdef OPENSSL
 
 #include "encryption/encryption.h"
 
-#include "util/string_util.h"
+#include <algorithm>
+#include <limits>
+
+#include <openssl/opensslv.h>
+
+#include "port/port.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace encryption {
 
-Status AESBlockCipher::InitKey(const std::string& key) {
-// AES_set_encrypt_key and AES_set_decrypt_key are deprecated: Since OpenSSL 3.0
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
- int ret =
- AES_set_encrypt_key(reinterpret_cast<const unsigned char*>(key.data()),
- static_cast<int>(key.size()) * 8, &encrypt_key_);
- if (ret != 0) {
- return Status::InvalidArgument("AES set encrypt key error: " +
- std::to_string(ret));
- }
- ret = AES_set_decrypt_key(reinterpret_cast<const unsigned char*>(key.data()),
- static_cast<int>(key.size()) * 8, &decrypt_key_);
-#pragma GCC diagnostic pop
- if (ret != 0) {
- return Status::InvalidArgument("AES set decrypt key error: " +
- std::to_string(ret));
+namespace {
+uint64_t GetBigEndian64(const unsigned char* buf) {
+ if (port::kLittleEndian) {
+ return (static_cast<uint64_t>(buf[0]) << 56) +
+ (static_cast<uint64_t>(buf[1]) << 48) +
+ (static_cast<uint64_t>(buf[2]) << 40) +
+ (static_cast<uint64_t>(buf[3]) << 32) +
+ (static_cast<uint64_t>(buf[4]) << 24) +
+ (static_cast<uint64_t>(buf[5]) << 16) +
+ (static_cast<uint64_t>(buf[6]) << 8) +
+ (static_cast<uint64_t>(buf[7]));
+ } else {
+ return *(reinterpret_cast<const uint64_t*>(buf));
+ }
+}
+
+void PutBigEndian64(uint64_t value, unsigned char* buf) {
+ if (port::kLittleEndian) {
+ buf[0] = static_cast<unsigned char>((value >> 56) & 0xff);
+ buf[1] = static_cast<unsigned char>((value >> 48) & 0xff);
+ buf[2] = static_cast<unsigned char>((value >> 40) & 0xff);
+ buf[3] = static_cast<unsigned char>((value >> 32) & 0xff);
+ buf[4] = static_cast<unsigned char>((value >> 24) & 0xff);
+ buf[5] = static_cast<unsigned char>((value >> 16) & 0xff);
+ buf[6] = static_cast<unsigned char>((value >> 8) & 0xff);
+ buf[7] = static_cast<unsigned char>(value & 0xff);
+ } else {
+ *(reinterpret_cast<uint64_t*>(buf)) = value;
+ }
+}
+} // anonymous namespace
+
+// AESCTRCipherStream use OpenSSL EVP API with CTR mode to encrypt and decrypt
+// data, instead of using the CTR implementation provided by
+// BlockAccessCipherStream. Benefits:
+//
+// 1. The EVP API automatically figure out if AES-NI can be enabled.
+// 2. Keep the data format consistent with OpenSSL (e.g. how IV is interpreted
+// as block counter).
+//
+// References for the openssl EVP API:
+// * man page: https://www.openssl.org/docs/man1.1.1/man3/EVP_EncryptUpdate.html
+// * SO answer for random access: https://stackoverflow.com/a/57147140/11014942
+// *
+// https://medium.com/@amit.kulkarni/encrypting-decrypting-a-file-using-openssl-evp-b26e0e4d28d4
+Status AESCTRCipherStream::Cipher(uint64_t file_offset, char* data,
+ size_t data_size, bool is_encrypt) {
+#if OPENSSL_VERSION_NUMBER < 0x01000200f
+ (void)file_offset;
+ (void)data;
+ (void)data_size;
+ (void)is_encrypt;
+ return Status::NotSupported("OpenSSL version < 1.0.2");
+#else
+ int ret = 1;
+ EVP_CIPHER_CTX* ctx = nullptr;
+ InitCipherContext(ctx);
+ if (ctx == nullptr) {
+ return Status::IOError("Failed to create cipher context.");
+ }
+
+ uint64_t block_index = file_offset / AES_BLOCK_SIZE;
+ uint64_t block_offset = file_offset % AES_BLOCK_SIZE;
+
+ // In CTR mode, OpenSSL EVP API treat the IV as a 128-bit big-endien, and
+ // increase it by 1 for each block.
+ //
+ // In case of unsigned integer overflow in c++, the result is moduloed by
+ // range, means only the lowest bits of the result will be kept.
+ // http://www.cplusplus.com/articles/DE18T05o/
+ uint64_t iv_high = initial_iv_high_;
+ uint64_t iv_low = initial_iv_low_ + block_index;
+ if (std::numeric_limits<uint64_t>::max() - block_index < initial_iv_low_) {
+ iv_high++;
+ }
+ unsigned char iv[AES_BLOCK_SIZE];
+ PutBigEndian64(iv_high, iv);
+ PutBigEndian64(iv_low, iv + sizeof(uint64_t));
+
+ ret = EVP_CipherInit(ctx, cipher_,
+ reinterpret_cast<const unsigned char*>(key_.data()), iv,
+ (is_encrypt ? 1 : 0));
+ if (ret != 1) {
+ return Status::IOError("Failed to init cipher.");
+ }
+
+ // Disable padding. After disabling padding, data size should always be
+ // multiply of block size.
+ ret = EVP_CIPHER_CTX_set_padding(ctx, 0);
+ if (ret != 1) {
+ return Status::IOError("Failed to disable padding for cipher context.");
+ }
+
+ uint64_t data_offset = 0;
+ size_t remaining_data_size = data_size;
+ int output_size = 0;
+ unsigned char partial_block[AES_BLOCK_SIZE];
+
+ // In the following we assume EVP_CipherUpdate allow in and out buffer are
+ // the same, to save one memcpy. This is not specified in official man page.
+
+ // Handle partial block at the beginning. The parital block is copied to
+ // buffer to fake a full block.
+ if (block_offset > 0) {
+ size_t partial_block_size =
+ std::min<size_t>(AES_BLOCK_SIZE - block_offset, remaining_data_size);
+ memcpy(partial_block + block_offset, data, partial_block_size);
+ ret = EVP_CipherUpdate(ctx, partial_block, &output_size, partial_block,
+ AES_BLOCK_SIZE);
+ if (ret != 1) {
+ return Status::IOError("Crypter failed for first block, offset " +
+ std::to_string(file_offset));
+ }
+ if (output_size != AES_BLOCK_SIZE) {
+ return Status::IOError(
+ "Unexpected crypter output size for first block, expected " +
+ std::to_string(AES_BLOCK_SIZE) + " vs actual " + std::to_string(output_size));
+ }
+ memcpy(data, partial_block + block_offset, partial_block_size);
+ data_offset += partial_block_size;
+ remaining_data_size -= partial_block_size;
  }
+
+ // Handle full blocks in the middle.
+ if (remaining_data_size >= AES_BLOCK_SIZE) {
+ size_t actual_data_size =
+ remaining_data_size - remaining_data_size % AES_BLOCK_SIZE;
+ unsigned char* full_blocks =
+ reinterpret_cast<unsigned char*>(data) + data_offset;
+ ret = EVP_CipherUpdate(ctx, full_blocks, &output_size, full_blocks,
+ static_cast<int>(actual_data_size));
+ if (ret != 1) {
+ return Status::IOError("Crypter failed at offset " +
+ std::to_string(file_offset + data_offset));
+ }
+ if (output_size != static_cast<int>(actual_data_size)) {
+ return Status::IOError("Unexpected crypter output size, expected " +
+ std::to_string(actual_data_size) + " vs actual " +
+ std::to_string(output_size));
+ }
+ data_offset += actual_data_size;
+ remaining_data_size -= actual_data_size;
+ }
+
+ // Handle partial block at the end. The parital block is copied to buffer to
+ // fake a full block.
+ if (remaining_data_size > 0) {
+ assert(remaining_data_size < AES_BLOCK_SIZE);
+ memcpy(partial_block, data + data_offset, remaining_data_size);
+ ret = EVP_CipherUpdate(ctx, partial_block, &output_size, partial_block,
+ AES_BLOCK_SIZE);
+ if (ret != 1) {
+ return Status::IOError("Crypter failed for last block, offset " +
+ std::to_string(file_offset + data_offset));
+ }
+ if (output_size != AES_BLOCK_SIZE) {
+ return Status::IOError(
+ "Unexpected crypter output size for last block, expected " +
+ std::to_string(AES_BLOCK_SIZE) + " vs actual " + std::to_string(output_size));
+ }
+ memcpy(data + data_offset, partial_block, remaining_data_size);
+ }
+ FreeCipherContext(ctx);
  return Status::OK();
+#endif
 }
 
 Status NewAESCTRCipherStream(EncryptionMethod method, const std::string& key,
  const std::string& iv,
  std::unique_ptr<AESCTRCipherStream>* result) {
  assert(result != nullptr);
- size_t key_size = KeySize(method);
- if (key_size == 0) {
- return Status::InvalidArgument("Unsupported encryption method: " +
- std::to_string(static_cast<int>(method)));
+ const EVP_CIPHER* cipher = nullptr;
+ switch (method) {
+ case EncryptionMethod::kAES128_CTR:
+ cipher = EVP_aes_128_ctr();
+ break;
+ case EncryptionMethod::kAES192_CTR:
+ cipher = EVP_aes_192_ctr();
+ break;
+ case EncryptionMethod::kAES256_CTR:
+ cipher = EVP_aes_256_ctr();
+ break;
+ default:
+ return Status::InvalidArgument("Unsupported encryption method: " +
+ std::to_string(static_cast<int>(method)));
  }
- if (key.size() != key_size) {
+ if (key.size() != KeySize(method)) {
  return Status::InvalidArgument("Encryption key size mismatch. " +
  std::to_string(key.size()) + "(actual) vs. " +
- std::to_string(key_size) + "(expected).");
+ std::to_string(KeySize(method)) + "(expected).");
  }
  if (iv.size() != AES_BLOCK_SIZE) {
  return Status::InvalidArgument(
  "iv size not equal to block cipher block size: " + std::to_string(iv.size()) +
  "(actual) vs. " + std::to_string(AES_BLOCK_SIZE) + "(expected).");
  }
- std::unique_ptr<AESCTRCipherStream> cipher_stream(new AESCTRCipherStream(iv));
- Status s = cipher_stream->InitKey(key);
- if (!s.ok()) {
-  return s;
- }
- *result = std::move(cipher_stream);
+ Slice iv_slice(iv);
+ uint64_t iv_high =
+  GetBigEndian64(reinterpret_cast<const unsigned char*>(iv.data()));
+ uint64_t iv_low = GetBigEndian64(
+  reinterpret_cast<const unsigned char*>(iv.data() + sizeof(uint64_t)));
+ result->reset(new AESCTRCipherStream(cipher, key, iv_high, iv_low));
  return Status::OK();
 }