From 93003290d3481a11cb6c4a1d26067a1892711c06 Mon Sep 17 00:00:00 2001
From: Yang Zhang <yangchuan.zy@alibaba-inc.com>
Date: Tue, 5 Mar 2024 08:05:02 -0800
Subject: [PATCH] Propagate HiveConfig::kLoadQuantum (#8947)

Summary:
The HiveConfig::kLoadQuantum setting should be propagated to readerOptions, enabling DirectBufferedInput to utilize it.

Pull Request resolved: https://github.com/facebookincubator/velox/pull/8947

Reviewed By: xiaoxmeng, pedroerp

Differential Revision: D54500003

Pulled By: mbasmanova

fbshipit-source-id: 73499e1bbbdc808a636d10dd3f2bac2d3787d597
---
 velox/connectors/hive/HiveConnectorUtil.cpp           | 1 +
 velox/connectors/hive/tests/HiveConnectorUtilTest.cpp | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/velox/connectors/hive/HiveConnectorUtil.cpp b/velox/connectors/hive/HiveConnectorUtil.cpp
index 93a15cfdfbce..c43eecd31fc6 100644
--- a/velox/connectors/hive/HiveConnectorUtil.cpp
+++ b/velox/connectors/hive/HiveConnectorUtil.cpp
@@ -448,6 +448,7 @@ void configureReaderOptions(
     const RowTypePtr& fileSchema,
     const std::shared_ptr<HiveConnectorSplit>& hiveSplit,
     const std::unordered_map<std::string, std::string>& tableParameters) {
+  readerOptions.setLoadQuantum(hiveConfig->loadQuantum());
   readerOptions.setMaxCoalesceBytes(hiveConfig->maxCoalescedBytes());
   readerOptions.setMaxCoalesceDistance(hiveConfig->maxCoalescedDistanceBytes());
   readerOptions.setFileColumnNamesReadAsLowerCase(
diff --git a/velox/connectors/hive/tests/HiveConnectorUtilTest.cpp b/velox/connectors/hive/tests/HiveConnectorUtilTest.cpp
index 84c5b28f6b82..8f4ae79cac68 100644
--- a/velox/connectors/hive/tests/HiveConnectorUtilTest.cpp
+++ b/velox/connectors/hive/tests/HiveConnectorUtilTest.cpp
@@ -101,6 +101,7 @@ TEST_F(HiveConnectorUtilTest, configureReaderOptions) {
   EXPECT_EQ(readerOptions.getFileFormat(), fileFormat);
   EXPECT_TRUE(
       compareSerDeOptions(readerOptions.getSerDeOptions(), expectedSerDe));
+  EXPECT_EQ(readerOptions.loadQuantum(), hiveConfig->loadQuantum());
   EXPECT_EQ(readerOptions.maxCoalesceBytes(), hiveConfig->maxCoalescedBytes());
   EXPECT_EQ(
       readerOptions.maxCoalesceDistance(),
@@ -174,6 +175,7 @@ TEST_F(HiveConnectorUtilTest, configureReaderOptions) {
   // Tests other custom reader options.
   clearDynamicParameters(FileFormat::TEXT);
   std::unordered_map<std::string, std::string> customHiveConfigProps;
+  customHiveConfigProps[hive::HiveConfig::kLoadQuantum] = "321";
   customHiveConfigProps[hive::HiveConfig::kMaxCoalescedBytes] = "129";
   customHiveConfigProps[hive::HiveConfig::kMaxCoalescedDistanceBytes] = "513";
   customHiveConfigProps[hive::HiveConfig::kFileColumnNamesReadAsLowerCase] =
@@ -184,6 +186,7 @@ TEST_F(HiveConnectorUtilTest, configureReaderOptions) {
   hiveConfig = std::make_shared<hive::HiveConfig>(
       std::make_shared<core::MemConfig>(customHiveConfigProps));
   performConfigure();
+  EXPECT_EQ(readerOptions.loadQuantum(), hiveConfig->loadQuantum());
   EXPECT_EQ(readerOptions.maxCoalesceBytes(), hiveConfig->maxCoalescedBytes());
   EXPECT_EQ(
       readerOptions.maxCoalesceDistance(),