Split the memory manager creation and get (#2)

Summary: Pull Request resolved: #2 Split the process wide memory manager creation and get. Deprecate the default memory manager and memory pool APIs. X-link: facebookincubator/velox#7168 Reviewed By: mbasmanova Differential Revision: D50513171 Pulled By: xiaoxmeng fbshipit-source-id: 0f147c1e3c971ea4f39271fe5609a2224971509d
facebookincubator · Dec 16, 2023 · 87cfd5d · 87cfd5d
1 parent c3dacad
commit 87cfd5d
Show file tree

Hide file tree

Showing 22 changed files with 46 additions and 43 deletions.
diff --git a/alpha/common/tests/HuffmanTests.cpp b/alpha/common/tests/HuffmanTests.cpp
@@ -8,7 +8,7 @@ using namespace ::facebook;
 class HuffmanTests : public ::testing::Test {
  protected:
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   }
 
   std::shared_ptr<facebook::velox::memory::MemoryPool> pool_;

diff --git a/alpha/common/tests/VectorTests.cpp b/alpha/common/tests/VectorTests.cpp
@@ -15,7 +15,7 @@ class VectorTests : public ::testing::Test {
   }
 
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   }
 
   std::shared_ptr<velox::memory::MemoryPool> pool_;

diff --git a/alpha/encodings/tests/ConstantEncodingTests.cpp b/alpha/encodings/tests/ConstantEncodingTests.cpp
@@ -18,7 +18,7 @@ template <typename C>
 class ConstantEncodingTest : public ::testing::Test {
  protected:
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
     buffer_ = std::make_unique<alpha::Buffer>(*pool_);
   }
 

diff --git a/alpha/encodings/tests/EncodingLayoutTests.cpp b/alpha/encodings/tests/EncodingLayoutTests.cpp
@@ -46,7 +46,7 @@ void testCapture(alpha::EncodingLayout expected, TCollection data) {
     return encodingFactory.createPolicy(dataType);
   };
 
-  auto defaultPool = velox::memory::addDefaultLeafMemoryPool();
+  auto defaultPool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*defaultPool};
   auto encoding = alpha::EncodingFactory::encode<T>(
       std::make_unique<alpha::ReplayedEncodingSelectionPolicy<T>>(
@@ -257,7 +257,7 @@ TEST(EncodingLayoutTests, Nullable) {
     return encodingFactory.createPolicy(dataType);
   };
 
-  auto defaultPool = velox::memory::addDefaultLeafMemoryPool();
+  auto defaultPool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*defaultPool};
   auto encoding = alpha::EncodingFactory::encodeNullable<uint32_t>(
       std::make_unique<alpha::ReplayedEncodingSelectionPolicy<uint32_t>>(

diff --git a/alpha/encodings/tests/EncodingSelectionTests.cpp b/alpha/encodings/tests/EncodingSelectionTests.cpp
@@ -92,7 +92,7 @@ typename alpha::TypeTraits<T>::physicalType asPhysicalType(T value) {
 
 template <typename T>
 void test(std::span<const T> values, std::vector<EncodingDetails> expected) {
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   auto policy = getRootManualSelectionPolicy<T>();
   alpha::Buffer buffer{*pool};
 
@@ -529,7 +529,7 @@ TEST(EncodingSelectionBoolTests, SelectTrivial) {
   policies.push_back(
       std::make_unique<alpha::LearnedEncodingSelectionPolicy<T>>());
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   for (auto& policy : policies) {
@@ -561,7 +561,7 @@ TEST(EncodingSelectionBoolTests, SelectRunLength) {
   LOG(INFO) << "seed: " << seed;
   std::mt19937 rng(seed);
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   std::vector<uint32_t> runLengths;
@@ -605,7 +605,7 @@ TEST(EncodingSelectionBoolTests, SelectRunLength) {
 TEST(EncodingSelectionStringTests, SelectConst) {
   using T = std::string_view;
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   for (const T value :
@@ -638,7 +638,7 @@ TEST(EncodingSelectionStringTests, SelectConst) {
 TEST(EncodingSelectionStringTests, SelectMainlyConst) {
   using T = std::string_view;
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   for (const T value : {
@@ -701,7 +701,7 @@ TEST(EncodingSelectionStringTests, SelectTrivial) {
   LOG(INFO) << "seed: " << seed;
   std::mt19937 rng(seed);
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   auto policy = getRootManualSelectionPolicy<T>();
@@ -750,7 +750,7 @@ TEST(EncodingSelectionStringTests, SelectDictionary) {
 
   std::vector<std::string> uniqueValues{"", "abcdef", std::string(5000, '\0')};
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   auto policy = getRootManualSelectionPolicy<T>();
@@ -796,7 +796,7 @@ TEST(EncodingSelectionStringTests, SelectRunLength) {
   LOG(INFO) << "seed: " << seed;
   std::mt19937 rng(seed);
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
 
   std::vector<uint32_t> runLengths;
@@ -856,7 +856,7 @@ TEST(EncodingSelectionStringTests, SelectRunLength) {
 
 TEST(EncodingSelectionTests, TestNullable) {
   using T = std::string_view;
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
   auto policy = getRootManualSelectionPolicy<T>();
   std::vector<T> data{"abcd", "efg", "hijk", "lmno"};

diff --git a/alpha/encodings/tests/EncodingTestsNew.cpp b/alpha/encodings/tests/EncodingTestsNew.cpp
@@ -127,7 +127,7 @@ class EncodingTests : public ::testing::Test {
   using E = typename C::cppDataType;
 
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
     buffer_ = std::make_unique<alpha::Buffer>(*this->pool_);
     util_ = std::make_unique<alpha::testing::Util>(*this->pool_);
   }

diff --git a/alpha/encodings/tests/MainlyConstantEncodingTests.cpp b/alpha/encodings/tests/MainlyConstantEncodingTests.cpp
@@ -18,7 +18,7 @@ template <typename C>
 class MainlyConstantEncodingTest : public ::testing::Test {
  protected:
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
     buffer_ = std::make_unique<alpha::Buffer>(*pool_);
   }
 

diff --git a/alpha/encodings/tests/NullableEncodingTests.cpp b/alpha/encodings/tests/NullableEncodingTests.cpp
@@ -35,7 +35,7 @@ class NullableEncodingTest : public ::testing::Test {
   using E = typename C::cppDataType;
 
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
     buffer_ = std::make_unique<alpha::Buffer>(*pool_);
     util_ = std::make_unique<alpha::testing::Util>(*pool_);
   }

diff --git a/alpha/encodings/tests/RleEncodingTests.cpp b/alpha/encodings/tests/RleEncodingTests.cpp
@@ -17,7 +17,7 @@ template <typename C>
 class RleEncodingTest : public ::testing::Test {
  protected:
   void SetUp() override {
-    pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+    pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
     buffer_ = std::make_unique<alpha::Buffer>(*pool_);
   }
 

diff --git a/alpha/encodings/tests/SentinelEncodingTests.cpp b/alpha/encodings/tests/SentinelEncodingTests.cpp
@@ -13,7 +13,7 @@
 // class SentinelEncodingTest : public ::testing::Test {
 //  protected:
 //   void SetUp() override {
-//     pool_ = facebook::velox::memory::addDefaultLeafMemoryPool();
+//     pool_ = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
 //     buffer_ = std::make_unique<alpha::Buffer>(*pool_);
 //   }
 

diff --git a/alpha/encodings/tests/TestGenerator.cpp b/alpha/encodings/tests/TestGenerator.cpp
@@ -126,7 +126,7 @@ void writeFile(
 
   LOG(INFO) << "Writing " << identifier;
 
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Buffer buffer{*pool};
   auto data = dataGenerator(rng, buffer, rowCount);
 
@@ -340,7 +340,7 @@ int main(int argc, char* argv[]) {
     std::vector<char> buffer(size);
     stream.read(buffer.data(), buffer.size());
 
-    auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+    auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
     auto encoding =
         alpha::EncodingFactory::decode(*pool, {buffer.data(), buffer.size()});
     auto rowCount = encoding->rowCount();

diff --git a/alpha/tablet/tests/TabletTests.cpp b/alpha/tablet/tests/TabletTests.cpp
@@ -249,7 +249,7 @@ void test(
 class TabletTestSuite : public ::testing::Test {
  protected:
   void SetUp() override {
-    pool_ = velox::memory::addDefaultLeafMemoryPool();
+    pool_ = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   }
 
   std::shared_ptr<velox::memory::MemoryPool> pool_;
@@ -477,7 +477,7 @@ TEST(TabletTests, OptionalSections) {
   LOG(INFO) << "seed: " << seed;
   std::mt19937 rng{seed};
 
-  auto pool = velox::memory::addDefaultLeafMemoryPool();
+  auto pool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   std::string file;
   velox::InMemoryWriteFile writeFile(&file);
   alpha::TabletWriter tabletWriter{*pool, &writeFile};
@@ -533,7 +533,7 @@ TEST(TabletTests, OptionalSections) {
 }
 
 TEST(TabletTests, OptionalSectionsEmpty) {
-  auto pool = velox::memory::addDefaultLeafMemoryPool();
+  auto pool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   std::string file;
   velox::InMemoryWriteFile writeFile(&file);
   alpha::TabletWriter tabletWriter{*pool, &writeFile};
@@ -554,7 +554,7 @@ TEST(TabletTests, OptionalSectionsPreload) {
 
   for (const auto footerCompressionThreshold :
        {0U, std::numeric_limits<uint32_t>::max()}) {
-    auto pool = velox::memory::addDefaultLeafMemoryPool();
+    auto pool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
     std::string file;
     velox::InMemoryWriteFile writeFile(&file);
     alpha::TabletWriter tabletWriter{*pool, &writeFile};

diff --git a/alpha/tools/AlphaDumpLib.cpp b/alpha/tools/AlphaDumpLib.cpp
@@ -246,7 +246,7 @@ auto commaSeparated(T value) {
 } // namespace
 
 AlphaDumpLib::AlphaDumpLib(std::ostream& ostream, const std::string& file)
-    : pool_{velox::memory::addDefaultLeafMemoryPool()},
+    : pool_{velox::memory::deprecatedAddDefaultLeafMemoryPool()},
       file_{dwio::file_system::FileSystem::openForRead(
           file,
           dwio::common::request::AccessDescriptorBuilder()

diff --git a/alpha/tools/AlphaPerfTuningUtils.cpp b/alpha/tools/AlphaPerfTuningUtils.cpp
@@ -132,7 +132,7 @@ std::unordered_map<std::string, std::vector<int64_t>> extractFeatureOrdering(
 std::unordered_map<std::string, std::vector<int64_t>> generateFeatureList(
     const std::string& path,
     const dwio::common::request::AccessDescriptor& accessDescriptor) {
-  auto pool = velox::memory::addDefaultLeafMemoryPool();
+  auto pool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::VeloxReader reader{
       *pool,
       dwio::file_system::FileSystem::openForRead(path, accessDescriptor),
@@ -181,7 +181,7 @@ std::unordered_map<std::string, std::vector<int64_t>> generateFeatureList(
   // TODO: batch size should actually approximate koski writer or
   // trabant patterns.
   constexpr uint64_t batchSize = 25;
-  auto rootPool = velox::memory::defaultMemoryManager().addRootPool(
+  auto rootPool = velox::memory::deprecatedDefaultMemoryManager().addRootPool(
       "alpha_perf_tuning_rewriteFile");
   auto readerPool = rootPool->addLeafChild("reader");
   auto writerPool = rootPool->addLeafChild("writer");
@@ -265,7 +265,7 @@ std::unordered_map<std::string, std::vector<int64_t>> generateFeatureList(
     const FeatureProjectionConfig& featureProjectionConfig,
     size_t batchSize,
     size_t iters) {
-  auto rootPool = velox::memory::defaultMemoryManager().addRootPool(
+  auto rootPool = velox::memory::deprecatedDefaultMemoryManager().addRootPool(
       "alpha_perf_tuning_readFile");
   auto pool = rootPool->addLeafChild("reader");
   folly::StreamingStats<time_t> cpuStats;

diff --git a/alpha/tools/AlphaTransform.cpp b/alpha/tools/AlphaTransform.cpp
@@ -273,8 +273,9 @@ int main(int argc, char* argv[]) {
       fmt::format("Unknown run mode: {}", FLAGS_mode));
 
   LOG(INFO) << "Alpha Transform - Mode: " << FLAGS_mode;
-  auto pool = facebook::velox::memory::defaultMemoryManager().addRootPool(
-      "alpha_transform");
+  auto pool =
+      facebook::velox::memory::deprecatedDefaultMemoryManager().addRootPool(
+          "alpha_transform");
   try {
     if (FLAGS_mode == "transform_interactive") {
       // Interactive mode - Running in Spark.

diff --git a/alpha/tools/DwrfTransform.cpp b/alpha/tools/DwrfTransform.cpp
@@ -118,7 +118,8 @@ int main(int argc, char* argv[]) {
   ALPHA_CHECK(
       FLAGS_mode == "transform" || FLAGS_mode == "transform_interactive",
       fmt::format("Unknown run mode: {}", FLAGS_mode));
-  auto pool = facebook::velox::memory::defaultMemoryManager().addRootPool();
+  auto pool =
+      facebook::velox::memory::deprecatedDefaultMemoryManager().addRootPool();
 
   LOG(INFO) << "DWRF Transform - Mode: " << FLAGS_mode;
 

diff --git a/alpha/tools/EncodingLayoutTrainerMain.cpp b/alpha/tools/EncodingLayoutTrainerMain.cpp
@@ -43,7 +43,7 @@ std::string compress(std::string_view data) {
 
 int main(int argc, char* argv[]) {
   auto init = facebook::init::InitFacebookLight{&argc, &argv};
-  auto pool = velox::memory::addDefaultLeafMemoryPool();
+  auto pool = velox::memory::deprecatedAddDefaultLeafMemoryPool();
 
   if (FLAGS_input_files.empty()) {
     LOG(ERROR) << "Missing input files";

diff --git a/alpha/tools/EncodingSelectionLogger.cpp b/alpha/tools/EncodingSelectionLogger.cpp
@@ -27,7 +27,7 @@ using namespace ::facebook;
 
 template <typename T>
 void logEncodingSelection(const std::vector<std::string>& source) {
-  auto pool = facebook::velox::memory::addDefaultLeafMemoryPool();
+  auto pool = facebook::velox::memory::deprecatedAddDefaultLeafMemoryPool();
   alpha::Vector<T> values{pool.get()};
   values.reserve(source.size());
   for (const auto& value : source) {

diff --git a/alpha/velox/tests/SerializationTests.cpp b/alpha/velox/tests/SerializationTests.cpp
@@ -16,14 +16,14 @@ using namespace facebook;
 using namespace facebook::alpha;
 
 namespace {
-auto rootPool =
-    velox::memory::defaultMemoryManager().addRootPool("serialization_tests");
+auto rootPool = velox::memory::deprecatedDefaultMemoryManager().addRootPool(
+    "serialization_tests");
 } // namespace
 
 class SerializationTests : public ::testing::Test {
  protected:
   void SetUp() override {
-    pool_ = velox::memory::addDefaultLeafMemoryPool();
+    pool_ = velox::memory::deprecatedAddDefaultLeafMemoryPool();
   }
 
   std::shared_ptr<velox::memory::MemoryPool> pool_;

diff --git a/alpha/velox/tests/TypeTests.cpp b/alpha/velox/tests/TypeTests.cpp
@@ -13,7 +13,8 @@
 using namespace ::facebook;
 
 namespace {
-auto rootPool = velox::memory::defaultMemoryManager().addRootPool("type_tests");
+auto rootPool =
+    velox::memory::deprecatedDefaultMemoryManager().addRootPool("type_tests");
 auto leafPool = rootPool -> addLeafChild("leaf");
 } // namespace
 

diff --git a/alpha/velox/tests/VeloxReaderTests.cpp b/alpha/velox/tests/VeloxReaderTests.cpp
@@ -27,8 +27,8 @@
 using namespace ::facebook;
 
 namespace {
-auto rootPool =
-    velox::memory::defaultMemoryManager().addRootPool("velox_reader_tests");
+auto rootPool = velox::memory::deprecatedDefaultMemoryManager().addRootPool(
+    "velox_reader_tests");
 auto leafPool = rootPool -> addLeafChild("leaf");
 struct VeloxMapGeneratorConfig {
   std::shared_ptr<const velox::RowType> rowType;
@@ -976,7 +976,7 @@ void writeAndVerify(
   auto selector = std::make_shared<velox::dwio::common::ColumnSelector>(type);
   // new pool with to limit already used memory and with tracking enabled
   auto leakDetectPool =
-      facebook::velox::memory::defaultMemoryManager().addRootPool(
+      facebook::velox::memory::deprecatedDefaultMemoryManager().addRootPool(
           "memory_leak_detect");
   auto readerPool = leakDetectPool->addLeafChild("reader_pool");
 

diff --git a/alpha/velox/tests/VeloxWriterTests.cpp b/alpha/velox/tests/VeloxWriterTests.cpp
@@ -20,8 +20,8 @@
 using namespace ::facebook;
 
 namespace {
-auto rootPool =
-    velox::memory::defaultMemoryManager().addRootPool("velox_writer_tests");
+auto rootPool = velox::memory::deprecatedDefaultMemoryManager().addRootPool(
+    "velox_writer_tests");
 auto leafPool = rootPool -> addLeafChild("leaf");
 } // namespace