core: switching env for encryption support.

This is to be contrasted to the preamble method in cockroachdb#20124.

This method is discussed in the `Custom env for encryption state`
section of the
[Encryption RFC](cockroachdb#19785)

When encryption is enabled, use a switching env that can redirect
each Env method to one of:
* base env for plaintext (same format as currently, no overhead)
* encrypted env (with or without preamble) for encrypted files

The switching env will hold the list of encrypted files to know which
env to pick.

Author: marc

c-deps/libroach/CMakeLists.txt

@@ -23,6 +23,7 @@ project(roachlib)
 add_library(roach
   db.cc
   encoding.cc
+  env_switching.cc
   eventlistener.cc
   protos/roachpb/data.pb.cc
   protos/roachpb/internal.pb.cc

c-deps/libroach/db.cc

@@ -33,6 +33,7 @@
 #include "protos/storage/engine/enginepb/mvcc.pb.h"
 #include "db.h"
 #include "encoding.h"
+#include "env_switching.h"
 #include "eventlistener.h"
 #include "keys.h"
 
@@ -89,6 +90,7 @@ struct DBEngine {
 };
 
 struct DBImpl : public DBEngine {
+  std::unique_ptr<rocksdb::Env> switching_env;
   std::unique_ptr<rocksdb::Env> memenv;
   std::unique_ptr<rocksdb::DB> rep_deleter;
   std::shared_ptr<rocksdb::Cache> block_cache;
@@ -98,8 +100,9 @@ struct DBImpl : public DBEngine {
   // and Env will be deleted when the DBImpl is deleted. It is ok to
   // pass NULL for the Env.
   DBImpl(rocksdb::DB* r, rocksdb::Env* m, std::shared_ptr<rocksdb::Cache> bc,
-    std::shared_ptr<DBEventListener> event_listener)
+    std::shared_ptr<DBEventListener> event_listener, rocksdb::Env* s_env)
       : DBEngine(r),
+        switching_env(s_env),
         memenv(m),
         rep_deleter(r),
         block_cache(bc),
@@ -1697,14 +1700,20 @@ DBStatus DBOpen(DBEngine **db, DBSlice dir, DBOptions db_opts) {
     options.env = memenv.get();
   }
 
+  std::unique_ptr<rocksdb::Env> switching_env;
+  if (db_opts.use_switching_env) {
+    switching_env.reset(NewSwitchingEnv(options.env));
+    options.env = switching_env.get();
+  }
+
   rocksdb::DB *db_ptr;
   rocksdb::Status status = rocksdb::DB::Open(options, ToString(dir), &db_ptr);
   if (!status.ok()) {
     return ToDBStatus(status);
   }
   *db = new DBImpl(db_ptr, memenv.release(),
       db_opts.cache != nullptr ? db_opts.cache->rep : nullptr,
-      event_listener);
+      event_listener, switching_env.release());
   return kSuccess;
 }
 

c-deps/libroach/env_switching.cc

@@ -0,0 +1,32 @@
+// Copyright 2017 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied.  See the License for the specific language governing
+// permissions and limitations under the License.
+
+#include "env_switching.h"
+
+// SwitchingEnv switches between a base_env (usually a Env::Default or MemEnv) and an encrypted env.
+//             SwitchingEnv (EnvWrapper)
+//               PLAIN   ENCRYPTED
+//                 |        |
+//                 |   encrypted_env (EnvWrapper)
+//                 |        |
+//               base_env (Env)
+// Any unimplemented methods are called on the base_env.
+class SwitchingEnv : public rocksdb::EnvWrapper {
+  public:
+   SwitchingEnv(rocksdb::Env* base_env) : rocksdb::EnvWrapper(base_env) { }
+};
+
+rocksdb::Env* NewSwitchingEnv(rocksdb::Env* base_env) {
+  return new SwitchingEnv(base_env ? base_env : rocksdb::Env::Default());
+}

c-deps/libroach/env_switching.h

@@ -0,0 +1,20 @@
+// Copyright 2017 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied.  See the License for the specific language governing
+// permissions and limitations under the License.
+
+#pragma once
+
+#include <rocksdb/env.h>
+
+// Returns a new SwitchingEnv using the passed-in env as the base.
+rocksdb::Env* NewSwitchingEnv(rocksdb::Env* base_env);

c-deps/libroach/include/libroach.h

@@ -71,6 +71,7 @@ typedef struct {
   bool logging_enabled;
   int num_cpu;
   int max_open_files;
+  bool use_switching_env;
 } DBOptions;
 
 // Create a new cache with the specified size.

pkg/storage/engine/rocksdb.go

@@ -68,6 +68,10 @@ var minWALSyncInterval = settings.RegisterDurationSetting(
 	0*time.Millisecond,
 )
 
+// TODO(mberhault): enterprise encryption flags will dictate the need for the
+// switching env format. Until then, use an env variable.
+var useSwitchingEnv = envutil.EnvOrDefaultBool("COCKROACH_USE_SWITCHING_ENV", false)
+
 //export rocksDBLog
 func rocksDBLog(s *C.char, n C.int) {
 	// Note that rocksdb logging is only enabled if log.V(3) is true
@@ -401,28 +405,40 @@ func (r *RocksDB) String() string {
 }
 
 func (r *RocksDB) open() error {
-	var ver storageVersion
+	var existingVersion, newVersion storageVersion
 	if len(r.cfg.Dir) != 0 {
 		log.Infof(context.TODO(), "opening rocksdb instance at %q", r.cfg.Dir)
 
 		// Check the version number.
 		var err error
-		if ver, err = getVersion(r.cfg.Dir); err != nil {
+		if existingVersion, err = getVersion(r.cfg.Dir); err != nil {
 			return err
 		}
-		if ver < versionMinimum || ver > versionCurrent {
+		if existingVersion < versionMinimum || existingVersion > versionCurrent {
 			// Instead of an error, we should call a migration if possible when
 			// one is needed immediately following the DBOpen call.
 			return fmt.Errorf("incompatible rocksdb data version, current:%d, on disk:%d, minimum:%d",
-				versionCurrent, ver, versionMinimum)
+				versionCurrent, existingVersion, versionMinimum)
+		}
+
+		if existingVersion != versionCurrent {
+			if useSwitchingEnv {
+				newVersion = versionCurrent
+			} else {
+				// If the switching env is not needed, use the older version. This allows downgrade to binaries unaware
+				// of versionSwitchingEnv.
+				// TODO(mberhault): once enough releases supporting versionSwitchingEnv have passed, we can upgrade
+				// to it without worry.
+				newVersion = versionBeta20160331
+			}
 		}
 	} else {
 		if log.V(2) {
 			log.Infof(context.TODO(), "opening in memory rocksdb instance")
 		}
 
 		// In memory dbs are always current.
-		ver = versionCurrent
+		existingVersion = versionCurrent
 	}
 
 	blockSize := envutil.EnvOrDefaultBytes("COCKROACH_ROCKSDB_BLOCK_SIZE", defaultBlockSize)
@@ -434,20 +450,21 @@ func (r *RocksDB) open() error {
 
 	status := C.DBOpen(&r.rdb, goToCSlice([]byte(r.cfg.Dir)),
 		C.DBOptions{
-			cache:           r.cache.cache,
-			block_size:      C.uint64_t(blockSize),
-			wal_ttl_seconds: C.uint64_t(walTTL),
-			logging_enabled: C.bool(log.V(3)),
-			num_cpu:         C.int(runtime.NumCPU()),
-			max_open_files:  C.int(maxOpenFiles),
+			cache:             r.cache.cache,
+			block_size:        C.uint64_t(blockSize),
+			wal_ttl_seconds:   C.uint64_t(walTTL),
+			logging_enabled:   C.bool(log.V(3)),
+			num_cpu:           C.int(runtime.NumCPU()),
+			max_open_files:    C.int(maxOpenFiles),
+			use_switching_env: C.bool(useSwitchingEnv),
 		})
 	if err := statusToError(status); err != nil {
 		return errors.Wrap(err, "could not open rocksdb instance")
 	}
 
-	// Update or add the version file if needed.
-	if ver < versionCurrent {
-		if err := writeVersionFile(r.cfg.Dir); err != nil {
+	// Update or add the version file if needed and if on-disk.
+	if len(r.cfg.Dir) != 0 && existingVersion != newVersion {
+		if err := writeVersionFile(r.cfg.Dir, newVersion); err != nil {
 			return err
 		}
 	}

pkg/storage/engine/rocksdb_test.go

@@ -225,8 +225,8 @@ func TestRocksDBOpenWithVersions(t *testing.T) {
 		{false, Version{}, ""},
 		{true, Version{versionCurrent}, ""},
 		{true, Version{versionMinimum}, ""},
-		{true, Version{-1}, "incompatible rocksdb data version, current:1, on disk:-1, minimum:0"},
-		{true, Version{2}, "incompatible rocksdb data version, current:1, on disk:2, minimum:0"},
+		{true, Version{-1}, "incompatible rocksdb data version, current:2, on disk:-1, minimum:0"},
+		{true, Version{3}, "incompatible rocksdb data version, current:2, on disk:3, minimum:0"},
 	}
 
 	for i, testCase := range testCases {

pkg/storage/engine/version.go

@@ -27,13 +27,14 @@ type storageVersion int
 const (
 	versionNoFile storageVersion = iota
 	versionBeta20160331
+	versionSwitchingEnv
 )
 
 const (
 	versionFilename     = "COCKROACHDB_VERSION"
 	versionFilenameTemp = "COCKROACHDB_VERSION_TEMP"
 	versionMinimum      = versionNoFile
-	versionCurrent      = versionBeta20160331
+	versionCurrent      = versionSwitchingEnv
 )
 
 // Version stores all the version information for all stores and is used as
@@ -67,11 +68,11 @@ func getVersion(dir string) (storageVersion, error) {
 	return ver.Version, nil
 }
 
-// writeVersionFile overwrites the version file to contain the latest version.
-func writeVersionFile(dir string) error {
+// writeVersionFile overwrites the version file to contain the specified version.
+func writeVersionFile(dir string, ver storageVersion) error {
 	tempFilename := filepath.Join(dir, versionFilenameTemp)
 	filename := getVersionFilename(dir)
-	b, err := json.Marshal(Version{versionCurrent})
+	b, err := json.Marshal(Version{ver})
 	if err != nil {
 		return err
 	}

pkg/storage/engine/version_test.go

@@ -48,7 +48,7 @@ func TestVersions(t *testing.T) {
 	}
 
 	// Write the current versions to the file.
-	if err := writeVersionFile(dir); err != nil {
+	if err := writeVersionFile(dir, versionCurrent); err != nil {
 		t.Fatal(err)
 	}
 	ver, err = getVersion(dir)