Upgrade to Arrow 4.0.0 (#198)

* Point at Arrow-4.0.0 vcpkg PR from Ian Cook. * Fix changed encryption.h path. Fix changed key_value_metadata.h path. Fix LogicalType Unknown enum renamed to Undefined. Fix changed GetKey() signature override. Remove ReadBatchSpaced deprecated functions. * Fix re2 linker errors. * Include latest arrow port commits * Fix parquet version string (has finally been updated, woohoo!) * Update vcpkg_version.txt Point at latest arrow port PR commit * Arrow 4.0.0 port has been merged back to vcpkg master, point to it
G-Research · Jun 17, 2021 · 653ec55 · 653ec55
1 parent 1a0af68
commit 653ec55
Show file tree

Hide file tree

Showing 21 changed files with 25 additions and 234 deletions.
diff --git a/cmake/FindBrotli.cmake b/cmake/FindBrotli.cmake
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -2,13 +2,13 @@ include(GenerateExportHeader)
 
 find_package(Arrow REQUIRED)
 find_package(Boost COMPONENTS filesystem regex system REQUIRED)
-find_package(Brotli REQUIRED)
+find_package(unofficial-brotli CONFIG REQUIRED)
 find_package(BZip2 REQUIRED)
-find_package(double-conversion CONFIG REQUIRED)
 find_package(gflags CONFIG REQUIRED)
 find_package(glog CONFIG REQUIRED)
 find_package(lz4 CONFIG REQUIRED)
 find_package(OpenSSL REQUIRED)
+find_package(re2 CONFIG REQUIRED)
 find_package(Snappy CONFIG REQUIRED)
 find_package(Thrift CONFIG REQUIRED)
 find_package(utf8proc REQUIRED)
@@ -86,17 +86,17 @@ include_directories(
 	${PROJECT_BINARY_DIR}
 	${ParquetCpp_INCLUDE_DIRS})
 
-target_link_libraries(ParquetSharpNative 
+target_link_libraries(ParquetSharpNative PRIVATE
 	${ParquetCpp_LIBRARIES}
 	${Arrow_LIBRARIES}
 	${Boost_LIBRARIES}
-	${Brotli_LIBRARIES}
+	unofficial::brotli::brotlidec-static unofficial::brotli::brotlienc-static unofficial::brotli::brotlicommon-static
 	BZip2::BZip2
 	${Crypto_LIBRARIES}
-	double-conversion::double-conversion
 	glog::glog
 	lz4::lz4
 	${SSL_LIBRARIES}
+	re2::re2
 	Snappy::snappy
 	thrift::thrift
 	${utf8proc_LIBRARIES}

diff --git a/cpp/ColumnDecryptionProperties.cpp b/cpp/ColumnDecryptionProperties.cpp
@@ -3,7 +3,7 @@
 #include "CString.h"
 #include "ExceptionInfo.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/ColumnDecryptionPropertiesBuilder.cpp b/cpp/ColumnDecryptionPropertiesBuilder.cpp
@@ -3,7 +3,7 @@
 #include "CString.h"
 #include "ExceptionInfo.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/ColumnEncryptionProperties.cpp b/cpp/ColumnEncryptionProperties.cpp
@@ -3,7 +3,7 @@
 #include "CString.h"
 #include "ExceptionInfo.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/ColumnEncryptionPropertiesBuilder.cpp b/cpp/ColumnEncryptionPropertiesBuilder.cpp
@@ -3,7 +3,7 @@
 #include "CString.h"
 #include "ExceptionInfo.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/Enums.cpp b/cpp/Enums.cpp
@@ -35,7 +35,7 @@ namespace
 		static_assert(Encoding::BYTE_STREAM_SPLIT == 9);
 		static_assert(Encoding::UNDEFINED == 10);
 
-		static_assert(LogicalType::Type::UNKNOWN == 0);
+		static_assert(LogicalType::Type::UNDEFINED == 0);
 		static_assert(LogicalType::Type::STRING == 1);
 		static_assert(LogicalType::Type::MAP == 2);
 		static_assert(LogicalType::Type::LIST == 3);

diff --git a/cpp/FileDecryptionProperties.cpp b/cpp/FileDecryptionProperties.cpp
@@ -5,7 +5,7 @@
 #include "ManagedAadPrefixVerifier.h"
 #include "ManagedDecryptionKeyRetriever.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/FileDecryptionPropertiesBuilder.cpp b/cpp/FileDecryptionPropertiesBuilder.cpp
@@ -5,7 +5,7 @@
 #include "ManagedAadPrefixVerifier.h"
 #include "ManagedDecryptionKeyRetriever.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/FileEncryptionProperties.cpp b/cpp/FileEncryptionProperties.cpp
@@ -3,7 +3,7 @@
 #include "CString.h"
 #include "ExceptionInfo.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/FileEncryptionPropertiesBuilder.cpp b/cpp/FileEncryptionPropertiesBuilder.cpp
@@ -3,7 +3,7 @@
 #include "CString.h"
 #include "ExceptionInfo.h"
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/KeyValueMetadata.cpp b/cpp/KeyValueMetadata.cpp
@@ -2,6 +2,7 @@
 #include "cpp/ParquetSharpExport.h"
 #include "ExceptionInfo.h"
 
+#include <arrow/util/key_value_metadata.h>
 #include <parquet/metadata.h>
 
 using namespace parquet;
@@ -76,7 +77,7 @@ extern "C"
 
 	PARQUETSHARP_EXPORT void KeyValueMetadata_Free_Entries(const std::shared_ptr<const KeyValueMetadata>* key_value_metadata, const char** keys, const char** values)
 	{
-		int64_t size = (*key_value_metadata)->size();
+		const int64_t size = (*key_value_metadata)->size();
 
 		for (int i = 0; i != size; ++i)
 		{

diff --git a/cpp/LogicalType.cpp b/cpp/LogicalType.cpp
@@ -111,11 +111,6 @@ extern "C"
 		TRYCATCH(*logical_type = new std::shared_ptr<const LogicalType>(LogicalType::None());)
 	}
 
-	PARQUETSHARP_EXPORT ExceptionInfo* LogicalType_Unknown(const std::shared_ptr<const LogicalType>** logical_type)
-	{
-		TRYCATCH(*logical_type = new std::shared_ptr<const LogicalType>(LogicalType::Unknown());)
-	}
-
 	// Typed properties
 	PARQUETSHARP_EXPORT ExceptionInfo* DecimalLogicalType_Precision(const std::shared_ptr<const DecimalLogicalType>* logical_type, int32_t* precision)
 	{

diff --git a/cpp/ManagedAadPrefixVerifier.h b/cpp/ManagedAadPrefixVerifier.h
@@ -1,7 +1,7 @@
 
 #pragma once
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 
 using namespace parquet;
 

diff --git a/cpp/ManagedDecryptionKeyRetriever.h b/cpp/ManagedDecryptionKeyRetriever.h
@@ -1,7 +1,7 @@
 
 #pragma once
 
-#include <parquet/encryption.h>
+#include <parquet/encryption/encryption.h>
 #include <stdexcept>
 
 using namespace parquet;
@@ -35,7 +35,7 @@ class ManagedDecryptionKeyRetriever final : public DecryptionKeyRetriever
 		free_gc_handle_(Handle);
 	}
 
-	std::string GetKey(const std::string& key_metadata) const override
+	std::string GetKey(const std::string& key_metadata) override
 	{
 		const char* exception = nullptr;
 		AesKey key;

diff --git a/cpp/TypedColumnReader.cpp b/cpp/TypedColumnReader.cpp
@@ -29,32 +29,6 @@ extern "C"
 				values_read);)																	\
 	}																							\
 																								\
-	PARQUETSHARP_EXPORT ExceptionInfo* TypedColumnReader_ReadBatchSpaced_##ParquetType(			\
-		std::shared_ptr<ColumnReader>* columnReader,											\
-		int64_t batch_size, 																	\
-		int16_t* def_levels, 																	\
-		int16_t* rep_levels,																	\
-		NativeType* values, 																	\
-		uint8_t* valid_bits,																	\
-		int64_t valid_bits_offset,																\
-		int64_t* levels_read,																	\
-		int64_t* values_read,																	\
-		int64_t* null_count,																	\
-		int64_t* return_value)																	\
-	{																							\
-        TRYCATCH(																				\
-			*levels_read = static_cast<ParquetType##Reader&>(**columnReader).ReadBatchSpaced(	\
-				batch_size,																		\
-				def_levels,																		\
-				rep_levels,																		\
-				values,																			\
-				valid_bits,																		\
-				valid_bits_offset,																\
-				levels_read,																	\
-				values_read,																	\
-				null_count);)																	\
-	}																							\
-																								\
 	PARQUETSHARP_EXPORT ExceptionInfo* TypedColumnReader_Skip_##ParquetType(					\
 		std::shared_ptr<ColumnReader>* columnReader,											\
 		int64_t num_rows_to_skip,																\

diff --git a/csharp.test/TestPhysicalTypeRoundtrip.cs b/csharp.test/TestPhysicalTypeRoundtrip.cs
@@ -118,14 +118,14 @@ private static void AssertReadRoundtrip(ResizableBuffer buffer, ExpectedColumn[]
 
             var numRows = expectedColumns.First().Values.Length;
 
-            Assert.AreEqual("parquet-cpp version 1.5.1-SNAPSHOT", fileMetaData.CreatedBy);
+            Assert.AreEqual("parquet-cpp-arrow version 4.0.0", fileMetaData.CreatedBy);
             Assert.AreEqual(new Dictionary<string, string> {{"case", "Test"}, {"Awesome", "true"}}, fileMetaData.KeyValueMetadata);
             Assert.AreEqual(expectedColumns.Length, fileMetaData.NumColumns);
             Assert.AreEqual(numRows, fileMetaData.NumRows);
             Assert.AreEqual(1, fileMetaData.NumRowGroups);
             Assert.AreEqual(1 + expectedColumns.Length, fileMetaData.NumSchemaElements);
             Assert.AreEqual(ParquetVersion.PARQUET_1_0, fileMetaData.Version);
-            Assert.AreEqual("parquet-cpp version 1.5.1", fileMetaData.WriterVersion.ToString());
+            Assert.AreEqual("parquet-cpp-arrow version 4.0.0", fileMetaData.WriterVersion.ToString());
 
             using var rowGroupReader = fileReader.RowGroup(0);
             var rowGroupMetaData = rowGroupReader.MetaData;

diff --git a/csharp.test/TestWriterProperties.cs b/csharp.test/TestWriterProperties.cs
@@ -15,7 +15,7 @@ public static void TestDefaultProperties()
         {
             var p = WriterProperties.GetDefaultWriterProperties();
 
-            Assert.AreEqual("parquet-cpp version 1.5.1-SNAPSHOT", p.CreatedBy);
+            Assert.AreEqual("parquet-cpp-arrow version 4.0.0", p.CreatedBy);
             Assert.AreEqual(Compression.Uncompressed, p.Compression(new ColumnPath("anypath")));
             Assert.AreEqual(int.MinValue, p.CompressionLevel(new ColumnPath("anypath")));
             Assert.AreEqual(1024*1024, p.DataPageSize);