Merge #27426

27426: libroach: disable core file when encryption is requested. r=mberhault a=mberhault

Set core size soft/max limits to 0 when encryption-at-rest is enabled.

This spits out a loud warning on stdout (similar to warning about lack
of AES instruction set support) when the `(set|get)rlimit` calls fail,
or when running on Windows.

Release note (enterprise change): disable core dumps when enabling encryption

Co-authored-by: marc <marc@cockroachlabs.com>

Author: craig[bot]

c-deps/libroach/ccl/crypto_utils.cc

@@ -11,6 +11,13 @@
 #include <cryptopp/hex.h>
 #include <cryptopp/osrng.h>
 #include <cryptopp/sha.h>
+#include "../fmt.h"
+
+#ifndef _WIN32
+#include <sys/resource.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#endif
 
 std::string HexString(const std::string& s) {
   std::string value;
@@ -61,3 +68,30 @@ rocksdb_utils::BlockCipher* NewAESEncryptCipher(const enginepbccl::SecretKey* ke
 }
 
 bool UsesAESNI() { return CryptoPP::UsesAESNI(); }
+
+rocksdb::Status DisableCoreFile() {
+#ifdef _WIN32
+  return rocksdb::Status::NotSupported("preventing crash reports is not supported on Windows");
+#else
+  // We can't use prlimit on OSX. Use setrlimit and getrlimit instead.
+  rlimit lim;
+  int ret = getrlimit(RLIMIT_CORE, &lim);
+  if (ret != 0) {
+    return rocksdb::Status::NotSupported(
+        fmt::StringPrintf("failed to get core size rlimit: %s", strerror(errno)));
+  }
+
+  if (lim.rlim_cur == 0 && lim.rlim_max == 0) {
+    return rocksdb::Status::OK();
+  }
+
+  rlimit new_lim = {0, 0};
+  ret = setrlimit(RLIMIT_CORE, &new_lim);
+  if (ret != 0) {
+    return rocksdb::Status::NotSupported(
+        fmt::StringPrintf("failed to set core size rlimit: %s", strerror(errno)));
+  }
+
+  return rocksdb::Status::OK();
+#endif
+}

c-deps/libroach/ccl/crypto_utils.h

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include <rocksdb/status.h>
 #include <string>
 #include "../rocksdbutils/env_encryption.h"
 #include "ccl/storageccl/engineccl/enginepbccl/key_registry.pb.h"
@@ -32,3 +33,7 @@ rocksdb_utils::BlockCipher* NewAESEncryptCipher(const enginepbccl::SecretKey* ke
 
 // Returns true if CryptoPP is using AES-NI.
 bool UsesAESNI();
+
+// DisableCoreFile sets the maximum size of a core file to 0. Returns success
+// if successfully called.
+rocksdb::Status DisableCoreFile();

c-deps/libroach/ccl/crypto_utils_test.cc

@@ -10,3 +10,27 @@
 #include "crypto_utils.h"
 
 TEST(CryptoUtils, HasAESNI) { EXPECT_TRUE(UsesAESNI()); }
+
+#ifdef _WIN32
+
+TEST(CryptoUtils, DisableCoreDumps) {
+  EXPECT_ERR(DisableCoreFile(), ".* not supported on Windows");
+}
+
+#else
+
+#include <sys/resource.h>
+#include <sys/time.h>
+#include <sys/types.h>
+
+TEST(CryptoUtils, DisableCoreDumps) {
+  rlimit lim = {1 << 10, 2 << 10};
+  ASSERT_EQ(0, setrlimit(RLIMIT_CORE, &lim));
+
+  EXPECT_OK(DisableCoreFile());
+  ASSERT_EQ(0, getrlimit(RLIMIT_CORE, &lim));
+  EXPECT_EQ(0, lim.rlim_cur);
+  EXPECT_EQ(0, lim.rlim_max);
+}
+
+#endif

c-deps/libroach/ccl/db.cc

@@ -119,13 +119,25 @@ rocksdb::Status DBOpenHook(std::shared_ptr<rocksdb::Logger> info_log, const std:
   // We have encryption options. Check whether the AES instruction set is supported.
   if (!UsesAESNI()) {
     // Shout loudly on standard out.
-    std::cout << std::endl
+    std::cerr << std::endl
               << "*** WARNING ***" << std::endl
               << "Encryption requested, but no AES instruction set detected" << std::endl
               << "Expect significant performance degradation!" << std::endl
               << std::endl;
   }
 
+  // Attempt to disable core dumps.
+  auto status = DisableCoreFile();
+  if (!status.ok()) {
+    // Shout loudly on standard out.
+    std::cerr << std::endl
+              << "*** WARNING ***" << std::endl
+              << "Encryption requested, but could not disable core dumps: " << status.getState()
+              << std::endl
+              << "Keys may be leaked in core dumps!" << std::endl
+              << std::endl;
+  }
+
   // The Go code sets the "file_registry" storage version if we specified encryption flags,
   // but let's double check anyway.
   if (!db_opts.use_file_registry) {
@@ -147,7 +159,7 @@ rocksdb::Status DBOpenHook(std::shared_ptr<rocksdb::Logger> info_log, const std:
   // NOTE: FileKeyManager uses the default env as the MemEnv can never have pre-populated files.
   FileKeyManager* store_key_manager = new FileKeyManager(
       rocksdb::Env::Default(), opts.key_files().current_key(), opts.key_files().old_key());
-  rocksdb::Status status = store_key_manager->LoadKeys();
+  status = store_key_manager->LoadKeys();
   if (!status.ok()) {
     delete store_key_manager;
     return status;