WIP: rocksdb-level encryption with support for multiple keys.

build/common.mk

@@ -229,13 +229,14 @@ $(REPO_ROOT)/build/variables.mk: $(REPO_ROOT)/.go-version $(VARIABLES_MAKEFILES)
 # common because both the root Makefile and protobuf.mk have C dependencies.
 
 C_DEPS_DIR := $(abspath $(REPO_ROOT)/c-deps)
+CRYPTOPP_SRC_DIR := $(C_DEPS_DIR)/cryptopp
 JEMALLOC_SRC_DIR := $(C_DEPS_DIR)/jemalloc
 PROTOBUF_SRC_DIR := $(C_DEPS_DIR)/protobuf
 ROCKSDB_SRC_DIR  := $(C_DEPS_DIR)/rocksdb
 SNAPPY_SRC_DIR   := $(C_DEPS_DIR)/snappy
 LIBROACH_SRC_DIR := $(C_DEPS_DIR)/libroach
 
-C_LIBS_SRCS := $(JEMALLOC_SRC_DIR) $(PROTOBUF_SRC_DIR) $(ROCKSDB_SRC_DIR) $(SNAPPY_SRC_DIR) $(LIBROACH_SRC_DIR)
+C_LIBS_SRCS := $(CRYPTOPP_SRC_DIR) $(JEMALLOC_SRC_DIR) $(PROTOBUF_SRC_DIR) $(ROCKSDB_SRC_DIR) $(SNAPPY_SRC_DIR) $(LIBROACH_SRC_DIR)
 
 HOST_TRIPLE := $(shell $$($(GO) env CC) -dumpmachine)
 
@@ -296,6 +297,7 @@ ifdef MINGW
 BUILD_DIR := $(shell cygpath -m $(BUILD_DIR))
 endif
 
+CRYPTOPP_DIR := $(BUILD_DIR)/cryptopp
 JEMALLOC_DIR := $(BUILD_DIR)/jemalloc
 PROTOBUF_DIR := $(BUILD_DIR)/protobuf
 ROCKSDB_DIR  := $(BUILD_DIR)/rocksdb$(STDMALLOC_SUFFIX)$(if $(ENABLE_ROCKSDB_ASSERTIONS),_assert)
@@ -305,7 +307,7 @@ LIBROACH_DIR := $(BUILD_DIR)/libroach
 PROTOC_DIR := $(GOPATH)/native/$(HOST_TRIPLE)/protobuf
 PROTOC 		 := $(PROTOC_DIR)/protoc
 
-C_LIBS_OSS = $(if $(USE_STDMALLOC),,libjemalloc) libprotobuf libsnappy librocksdb libroach
+C_LIBS_OSS = $(if $(USE_STDMALLOC),,libjemalloc) libcryptopp libprotobuf libsnappy librocksdb libroach
 C_LIBS_CCL = $(C_LIBS_OSS) libroachccl
 
 # Go does not permit dashes in build tags. This is undocumented. Fun!
@@ -347,7 +349,7 @@ $(CGO_FLAGS_FILES): $(REPO_ROOT)/build/common.mk
 	@echo 'package $(notdir $(@D))' >> $@
 	@echo >> $@
 	@echo '// #cgo CPPFLAGS: -I$(JEMALLOC_DIR)/include' >> $@
-	@echo '// #cgo LDFLAGS: $(addprefix -L,$(PROTOBUF_DIR) $(JEMALLOC_DIR)/lib $(SNAPPY_DIR) $(ROCKSDB_DIR) $(LIBROACH_DIR))' >> $@
+	@echo '// #cgo LDFLAGS: $(addprefix -L,$(CRYPTOPP_DIR) $(PROTOBUF_DIR) $(JEMALLOC_DIR)/lib $(SNAPPY_DIR) $(ROCKSDB_DIR) $(LIBROACH_DIR))' >> $@
 	@echo 'import "C"' >> $@
 
 # BUILD ARTIFACT CACHING
@@ -369,6 +371,12 @@ $(CGO_FLAGS_FILES): $(REPO_ROOT)/build/common.mk
 # only rebuild the affected objects, but in practice dependencies on configure
 # flags are not tracked correctly, and these stale artifacts can cause
 # particularly hard-to-debug errors.
+$(CRYPTOPP_DIR)/Makefile: $(C_DEPS_DIR)/cryptopp-rebuild $(BOOTSTRAP_TARGET)
+	rm -rf $(CRYPTOPP_DIR)
+	mkdir -p $(CRYPTOPP_DIR)
+	@# NOTE: If you change the CMake flags below, bump the version in
+	@# $(C_DEPS_DIR)/cryptopp-rebuild. See above for rationale.
+	cd $(CRYPTOPP_DIR) && cmake $(CMAKE_FLAGS) $(CRYPTOPP_SRC_DIR)
 
 $(JEMALLOC_SRC_DIR)/configure.ac: $(BOOTSTRAP_TARGET)
 
@@ -437,6 +445,10 @@ $(LIBROACH_DIR)/Makefile: $(C_DEPS_DIR)/libroach-rebuild $(BOOTSTRAP_TARGET)
 $(PROTOC): $(PROTOC_DIR)/Makefile .ALWAYS_REBUILD
 	@$(MAKE) --no-print-directory -C $(PROTOC_DIR) protoc
 
+.PHONY: libcryptopp
+libcryptopp: $(CRYPTOPP_DIR)/Makefile
+	@$(MAKE) --no-print-directory -C $(CRYPTOPP_DIR) static
+
 .PHONY: libjemalloc
 libjemalloc: $(JEMALLOC_DIR)/Makefile
 	@$(MAKE) --no-print-directory -C $(JEMALLOC_DIR) build_lib_static
@@ -468,17 +480,21 @@ libroachccl: $(LIBROACH_DIR)/Makefile libroach
 
 .PHONY: clean-c-deps
 clean-c-deps:
+	rm -rf $(CRYPTOPP_DIR)
 	rm -rf $(JEMALLOC_DIR)
 	rm -rf $(PROTOBUF_DIR)
 	rm -rf $(ROCKSDB_DIR)
 	rm -rf $(SNAPPY_DIR)
+	rm -rf $(LIBROACH_DIR)
 
 .PHONY: unsafe-clean-c-deps
 unsafe-clean-c-deps:
+	git -C $(CRYPTOPP_SRC_DIR) clean -dxf
 	git -C $(JEMALLOC_SRC_DIR) clean -dxf
 	git -C $(PROTOBUF_SRC_DIR) clean -dxf
 	git -C $(ROCKSDB_SRC_DIR)  clean -dxf
 	git -C $(SNAPPY_SRC_DIR)   clean -dxf
+	git -C $(LIBROACH_SRC_DIR)   clean -dxf
 
 .SECONDEXPANSION:
 $(LOCAL_BIN)/%: $$(shell find $(PKG_ROOT)/cmd/$$*) | submodules

build/variables.mk

@@ -22,6 +22,8 @@ define VALID_VARS
   CMAKE_SYSTEM_NAME
   COCKROACH
   CONFIGURE_FLAGS
+  CRYPTOPP_DIR
+  CRYPTOPP_SRC_DIR
   CXX_PATH
   C_DEPS_DIR
   C_LIBS_CCL

c-deps/cryptopp-rebuild

@@ -0,0 +1 @@
+1

c-deps/libroach-rebuild

@@ -1,4 +1,4 @@
 Bump the version below when changing libroach CMake flags. Search for "BUILD
 ARTIFACT CACHING" in build/common.mk for rationale.
 
-1
+3

c-deps/libroach/CMakeLists.txt

@@ -23,6 +23,7 @@ project(roachlib)
 add_library(roach
   db.cc
   encoding.cc
+  encryption.cc
   eventlistener.cc
   protos/roachpb/data.pb.cc
   protos/roachpb/internal.pb.cc
@@ -36,7 +37,7 @@ add_library(roach
 )
 target_include_directories(roach
   PUBLIC  ./include
-  PRIVATE ../protobuf/src ../rocksdb/include protos
+  PRIVATE ../protobuf/src ../rocksdb/include ../cryptopp protos
 )
 
 add_library(roachccl

c-deps/libroach/db.cc

@@ -33,6 +33,7 @@
 #include "protos/storage/engine/enginepb/mvcc.pb.h"
 #include "db.h"
 #include "encoding.h"
+#include "encryption.h"
 #include "eventlistener.h"
 #include "keys.h"
 
@@ -1695,6 +1696,8 @@ DBStatus DBOpen(DBEngine **db, DBSlice dir, DBOptions db_opts) {
   if (dir.len == 0) {
     memenv.reset(rocksdb::NewMemEnv(rocksdb::Env::Default()));
     options.env = memenv.get();
+  } else {
+    options.env = GetEncryptedEnv(options.env);
   }
 
   rocksdb::DB *db_ptr;

c-deps/libroach/encryption.cc

@@ -0,0 +1,120 @@
+// Copyright 2017 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied.  See the License for the specific language governing
+// permissions and limitations under the License.
+
+#include <rocksdb/env.h>
+#include "encryption.h"
+
+#include <iostream>
+#include <osrng.h>
+#include <modes.h>
+#include <filters.h>
+#include <sha.h>
+#include <hex.h>
+
+CipherList ciphers;
+rocksdb::Env* GetEncryptedEnv(rocksdb::Env* base_env) {
+  // Make a few hard-coded keys and corresponding ciphers.
+  char *key1 = new char[CryptoPP::AES::DEFAULT_KEYLENGTH];
+  memset(key1, 0x01, CryptoPP::AES::DEFAULT_KEYLENGTH);
+  char *key2 = new char[CryptoPP::AES::DEFAULT_KEYLENGTH];
+  memset(key2, 0x02, CryptoPP::AES::DEFAULT_KEYLENGTH);
+
+  ciphers[1] = new AESCipher(key1);
+  ciphers[2] = new AESCipher(key2);
+  rocksdb::EncryptionProvider* provider = new CTREncryptionProviderWithKey(ciphers);
+  return rocksdb::NewEncryptedEnv(base_env ? base_env : rocksdb::Env::Default(), provider);
+}
+
+inline uint64_t DecodeFixed64(const char* ptr) {
+    uint64_t result;
+    memcpy(&result, ptr, sizeof(result));  // gcc optimizes this to a plain load
+    return result;
+}
+
+inline void EncodeFixed64(char* ptr, uint64_t entry) {
+    memcpy(ptr, &entry, sizeof(entry));
+}
+
+// GetPrefixLength returns the length of the prefix that is added to every file
+// and used for storing encryption options.
+// For optimal performance, the prefix length should be a multiple of 
+// the a page size.
+size_t CTREncryptionProviderWithKey::GetPrefixLength() {
+  return defaultPrefixLength;
+}
+
+// encodeCTRParameters encodes the parameters into the prefix and returns the pseudo-random IV and counter.
+static void encodeCTRParameters(char *prefix, size_t blockSize, uint64_t keyID, uint64_t &initialCounter, rocksdb::Slice &iv) {
+  // Zero-out the prefix.
+  memset(prefix, 0, defaultPrefixLength);
+
+  // Skip the first 64 bits, just zeros for now.
+  auto offset = sizeof(keyID);
+  // Second set of 64 bits is the key ID.
+  EncodeFixed64(prefix + offset, keyID);
+  // Add 2*blocksize worth of pseudo-random data.
+  offset += sizeof(keyID);
+  CryptoPP::OS_GenerateRandomBlock(false, (byte *)(prefix + offset), blockSize * 2);
+  // Extract the block-wise long initial counter.
+  initialCounter = DecodeFixed64(prefix + offset);
+  // Followed by a blockSize-long IV.
+  offset += blockSize;
+  iv = rocksdb::Slice(prefix + offset, blockSize);
+}
+
+// CreateNewPrefix initialized an allocated block of prefix memory 
+// for a new file.
+rocksdb::Status CTREncryptionProviderWithKey::CreateNewPrefix(const std::string& fname, char *prefix, size_t prefixLength) {
+  std::cout << "#### Creating new file: " << fname << " with active_key_: " << active_key_ << "\n";
+
+  // Take random data to extract initial counter & IV
+  auto blockSize = ciphers_[active_key_]->BlockSize();
+  uint64_t initialCounter;
+  rocksdb::Slice prefixIV;
+  encodeCTRParameters(prefix, blockSize, active_key_, initialCounter, prefixIV);
+
+  return rocksdb::Status::OK();
+}
+
+rocksdb::Status CTREncryptionProviderWithKey::CreateCipherStream(const std::string& fname, const rocksdb::EnvOptions& options, rocksdb::Slice &prefix, std::unique_ptr<rocksdb::BlockAccessCipherStream>* result) {
+  // Read plain text part of prefix.
+  uint64_t keyID, initialCounter;
+  rocksdb::Slice iv;
+
+  // Skip the 64 bits, just zeros for now.
+  auto offset = sizeof(keyID);
+  // Second set of 64 bits if the key ID.
+  keyID = DecodeFixed64(prefix.data() + offset);
+  // Then comes a blockSize-long initial counter (but we just want 64 bits).
+  offset += sizeof(keyID);
+  initialCounter = DecodeFixed64(prefix.data() + offset);
+  // Followed by a blockSize-long IV.
+  // TODO(marc): fail on missing key.
+  auto blockSize = ciphers_[keyID]->BlockSize();
+  offset += blockSize;
+  iv = rocksdb::Slice(prefix.data() + offset, blockSize);
+
+  std::cout << "#### Opening file: " << fname << " with key: " << keyID << "\n";
+
+  // Create cipher stream 
+  return CreateCipherStreamFromPrefix(fname, options, keyID, initialCounter, iv, prefix, result);
+}
+
+// CreateCipherStreamFromPrefix creates a block access cipher stream for a file given
+// given name and options. The given prefix is already decrypted.
+rocksdb::Status CTREncryptionProviderWithKey::CreateCipherStreamFromPrefix(const std::string& fname, const rocksdb::EnvOptions& options,
+    uint64_t keyID, uint64_t initialCounter, const rocksdb::Slice& iv, const rocksdb::Slice& prefix, std::unique_ptr<rocksdb::BlockAccessCipherStream>* result) {
+  (*result) = std::unique_ptr<rocksdb::BlockAccessCipherStream>(new rocksdb::CTRCipherStream(*ciphers_[keyID], iv.data(), initialCounter));
+  return rocksdb::Status::OK();
+}

c-deps/libroach/encryption.h

@@ -0,0 +1,100 @@
+// Copyright 2017 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied.  See the License for the specific language governing
+// permissions and limitations under the License.
+
+#ifndef ROACHLIB_ENCRYPTION_H
+#define ROACHLIB_ENCRYPTION_H
+
+#include <map>
+#include <aes.h>
+#include <modes.h>
+#include <rocksdb/env_encryption.h>
+
+const static size_t defaultPrefixLength = 4096;
+
+rocksdb::Env* GetEncryptedEnv(rocksdb::Env* base_env);
+
+// Implements a BlockCipher using AES-128.
+class AESCipher : public rocksdb::BlockCipher {
+    private:
+      size_t blockSize_;
+      CryptoPP::AES::Encryption enc_;
+      CryptoPP::AES::Decryption dec_;
+    public:
+      AESCipher(char *key) :
+        enc_((byte*)key, CryptoPP::AES::DEFAULT_KEYLENGTH),
+        dec_((byte*)key, CryptoPP::AES::DEFAULT_KEYLENGTH) {
+      }
+      virtual ~AESCipher() {}
+
+      // BlockSize returns the size of each block supported by this cipher stream.
+      virtual size_t BlockSize() override {
+        return CryptoPP::AES::BLOCKSIZE;
+      }
+
+      // Encrypt a block of data.
+      // Length of data is equal to BlockSize().
+      virtual rocksdb::Status Encrypt(char *data) override {
+        enc_.ProcessBlock((byte *)data);
+        return rocksdb::Status::OK();
+      }
+
+      // Decrypt a block of data.
+      // Length of data is equal to BlockSize().
+      virtual rocksdb::Status Decrypt(char *data) override {
+        dec_.ProcessBlock((byte *)data);
+        return rocksdb::Status::OK();
+      }
+};
+
+typedef std::map<uint64_t, rocksdb::BlockCipher*> CipherList;
+
+class CTREncryptionProviderWithKey : public rocksdb::EncryptionProvider {
+  private:
+      CipherList& ciphers_;
+      uint64_t active_key_;
+
+  public:
+     CTREncryptionProviderWithKey(CipherList& ciphers)
+        : ciphers_(ciphers) {
+          auto iter = ciphers_.rbegin();
+          if (iter == ciphers_.rend()) {
+            active_key_ = 0;
+          } else {
+            active_key_ = iter->first;
+          }
+        };
+    virtual ~CTREncryptionProviderWithKey() {}
+
+    // GetPrefixLength returns the length of the prefix that is added to every file
+    // and used for storing encryption options.
+    // For optimal performance, the prefix length should be a multiple of 
+    // the a page size.
+    virtual size_t GetPrefixLength() override;
+
+    // CreateNewPrefix initialized an allocated block of prefix memory 
+    // for a new file.
+    virtual rocksdb::Status CreateNewPrefix(const std::string& fname, char *prefix, size_t prefixLength) override;
+
+    // CreateCipherStream creates a block access cipher stream for a file given
+    // given name and options.
+    virtual rocksdb::Status CreateCipherStream(const std::string& fname, const rocksdb::EnvOptions& options,
+      rocksdb::Slice& prefix, std::unique_ptr<rocksdb::BlockAccessCipherStream>* result) override;
+  protected:
+    // CreateCipherStreamFromPrefix creates a block access cipher stream for a file given
+    // given name and options. The given prefix is already decrypted.
+    virtual rocksdb::Status CreateCipherStreamFromPrefix(const std::string& fname, const rocksdb::EnvOptions& options,
+      uint64_t keyID, uint64_t initialCounter, const rocksdb::Slice& iv, const rocksdb::Slice& prefix, std::unique_ptr<rocksdb::BlockAccessCipherStream>* result);
+};
+
+#endif // ROACHLIB_ENCRYPTION_H

pkg/ccl/storageccl/engineccl/rocksdb.go

@@ -22,6 +22,7 @@ import (
 // #cgo CPPFLAGS: -I../../../../c-deps/libroach/include
 // #cgo LDFLAGS: -lroachccl
 // #cgo LDFLAGS: -lroach
+// #cgo LDFLAGS: -lcryptopp
 // #cgo LDFLAGS: -lprotobuf
 // #cgo LDFLAGS: -lrocksdb
 // #cgo LDFLAGS: -lsnappy

pkg/storage/engine/rocksdb.go

@@ -51,6 +51,7 @@ import (
 
 // #cgo CPPFLAGS: -I../../../c-deps/libroach/include
 // #cgo LDFLAGS: -lroach
+// #cgo LDFLAGS: -lcryptopp
 // #cgo LDFLAGS: -lprotobuf
 // #cgo LDFLAGS: -lrocksdb
 // #cgo LDFLAGS: -lsnappy