[WIP] ARROW-22: Add schema adapter routines for converting Parquet schema metadata to in-memory Arrow schemas

cpp/CMakeLists.txt

@@ -378,6 +378,16 @@ function(ADD_ARROW_TEST_DEPENDENCIES REL_TEST_NAME)
  add_dependencies(${TEST_NAME} ${ARGN})
 endfunction()
 
+# A wrapper for target_link_libraries() that is compatible with NO_TESTS.
+function(ARROW_TEST_LINK_LIBRARIES REL_TEST_NAME)
+ if(NO_TESTS)
+ return()
+ endif()
+ get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+ target_link_libraries(${TEST_NAME} ${ARGN})
+endfunction()
+
 enable_testing()
 
 ############################################################
@@ -528,6 +538,7 @@ set(ARROW_SRCS
  src/arrow/ipc/metadata-internal.cc
 
  src/arrow/types/construct.cc
+ src/arrow/types/decimal.cc
  src/arrow/types/json.cc
  src/arrow/types/list.cc
  src/arrow/types/primitive.cc

cpp/src/arrow/parquet/CMakeLists.txt

@@ -19,9 +19,12 @@
 # arrow_parquet : Arrow <-> Parquet adapter
 
 set(PARQUET_SRCS
+ schema.cc
 )
 
 set(PARQUET_LIBS
+ arrow
+ ${PARQUET_SHARED_LIB}
 )
 
 add_library(arrow_parquet STATIC
@@ -30,6 +33,9 @@ add_library(arrow_parquet STATIC
 target_link_libraries(arrow_parquet ${PARQUET_LIBS})
 SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
 
+ADD_ARROW_TEST(parquet-schema-test)
+ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
+
 # Headers: top level
 install(FILES
  DESTINATION include/arrow/parquet)

cpp/src/arrow/parquet/parquet-schema-test.cc

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/parquet/schema.h"
+
+namespace arrow {
+
+namespace parquet {
+
+using parquet_cpp::Repetition;
+using parquet_cpp::schema::NodePtr;
+using parquet_cpp::schema::PrimitiveNode;
+
+TEST(TestNodeConversion, Primitive) {
+ NodePtr node = PrimitiveNode::Make("boolean", Repetition::REQUIRED,
+ parquet_cpp::Type::BOOLEAN);
+ std::shared_ptr<Field> field = NodeToField(node);
+ ASSERT_EQ(field->name, "boolean");
+ ASSERT_TRUE(field->type->Equals(std::make_shared<BooleanType>()));
+ ASSERT_FALSE(field->nullable);
+
+ node = PrimitiveNode::Make("int32", Repetition::REQUIRED, parquet_cpp::Type::INT32);
+ field = NodeToField(node);
+ ASSERT_EQ(field->name, "int32");
+ ASSERT_TRUE(field->type->Equals(std::make_shared<Int32Type>()));
+ ASSERT_FALSE(field->nullable);
+
+ node = PrimitiveNode::Make("int64", Repetition::REQUIRED, parquet_cpp::Type::INT64);
+ field = NodeToField(node);
+ ASSERT_EQ(field->name, "int64");
+ ASSERT_TRUE(field->type->Equals(std::make_shared<Int64Type>()));
+ ASSERT_FALSE(field->nullable);
+
+ // case parquet_cpp::Type::INT96:
+ // TODO: Implement!
+ // node = PrimitiveNode::Make("int96", Repetition::REQUIRED, parquet_cpp::Type::INT96);
+ // field = NodeToField(node);
+ // TODO: Assertions
+
+ // case parquet_cpp::Type::FLOAT:
+ node = PrimitiveNode::Make("float", Repetition::REQUIRED, parquet_cpp::Type::FLOAT);
+ field = NodeToField(node);
+ ASSERT_EQ(field->name, "float");
+ ASSERT_TRUE(field->type->Equals(std::make_shared<FloatType>()));
+ ASSERT_FALSE(field->nullable);
+
+ // case parquet_cpp::Type::DOUBLE:
+ node = PrimitiveNode::Make("double", Repetition::REQUIRED, parquet_cpp::Type::DOUBLE);
+ field = NodeToField(node);
+ ASSERT_EQ(field->name, "double");
+ ASSERT_TRUE(field->type->Equals(std::make_shared<DoubleType>()));
+ ASSERT_FALSE(field->nullable);
+
+ // TODO: Implement!
+ // node = PrimitiveNode::Make("byte_array", Repetition::REQUIRED,
+ // parquet_cpp::Type::BYTE_ARRAY);
+ // field = NodeToField(node);
+ // TODO: Assertions
+
+ // TODO: Implement!
+ // node = PrimitiveNode::Make("fixed_len_byte_array", Repetition::REQUIRED,
+ // parquet_cpp::Type::FIXED_LEN_BYTE_ARRAY);
+ // field = NodeToField(node);
+ // TODO: Assertions
+}
+
+TEST(TestNodeConversion, Logical) {
+}
+
+TEST(TestSchemaConversion, Basics) {
+}
+
+} // namespace parquet
+
+} // namespace arrow

cpp/src/arrow/parquet/schema.cc

@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <vector>
+
+#include "arrow/parquet/schema.h"
+#include "arrow/types/decimal.h"
+
+using parquet_cpp::schema::Node;
+using parquet_cpp::schema::NodePtr;
+using parquet_cpp::schema::GroupNode;
+using parquet_cpp::schema::PrimitiveNode;
+
+namespace arrow {
+
+namespace parquet {
+
+
+TypePtr MakeDecimalType(const PrimitiveNode* node) {
+ int precision = node->decimal_metadata().precision;
+ int scale = node->decimal_metadata().scale;
+ return TypePtr(new DecimalType(precision, scale));
+}
+
+// TODO: Logical Type Handling
+std::shared_ptr<Field> NodeToField(const NodePtr& node) {
+ TypePtr type;
+
+ if (node->is_group()) {
+ const GroupNode* group = static_cast<const GroupNode*>(node.get());
+ std::vector<std::shared_ptr<Field>> fields;
+ for (int i = 0; i < group->field_count(); i++) {
+ fields.push_back(NodeToField(group->field(i)));
+ }
+ type = TypePtr(new StructType(fields));
+ } else {
+ // Primitive (leaf) node
+ const PrimitiveNode* primitive = static_cast<const PrimitiveNode*>(node.get());
+
+ switch (primitive->physical_type()) {
+ case parquet_cpp::Type::BOOLEAN:
+ type = TypePtr(new BooleanType());
+ break;
+ case parquet_cpp::Type::INT32:
+ type = TypePtr(new Int32Type());
+ break;
+ case parquet_cpp::Type::INT64:
+ type = TypePtr(new Int64Type());
+ break;
+ case parquet_cpp::Type::INT96:
+ // TODO: Do we have that type in Arrow?
+ // type = TypePtr(new Int96Type());
+ break;
+ case parquet_cpp::Type::FLOAT:
+ type = TypePtr(new FloatType());
+ break;
+ case parquet_cpp::Type::DOUBLE:
+ type = TypePtr(new DoubleType());
+ break;
+ case parquet_cpp::Type::BYTE_ARRAY:
+ // TODO: Do we have that type in Arrow?
+ // type = TypePtr(new Int96Type());
+ break;
+ case parquet_cpp::Type::FIXED_LEN_BYTE_ARRAY:
+ switch (primitive->logical_type()) {
+ case parquet_cpp::LogicalType::DECIMAL:
+ type = MakeDecimalType(primitive);
+ break;
+ default:
+ // TODO: Do we have that type in Arrow?
+ break;
+ }
+ break;
+ }
+ }
+
+ if (node->is_repeated()) {
+ type = TypePtr(new ListType(type));
+ }
+
+ return std::shared_ptr<Field>(new Field(node->name(), type, !node->is_required()));
+}
+
+std::shared_ptr<Schema> FromParquetSchema(
+ const parquet_cpp::SchemaDescriptor* parquet_schema) {
+ std::vector<std::shared_ptr<Field>> fields;
+ const GroupNode* schema_node = static_cast<const GroupNode*>(
+ parquet_schema->schema().get());
+
+ // TODO: What to with the head node?
+ for (int i = 0; i < schema_node->field_count(); i++) {
+ fields.push_back(NodeToField(schema_node->field(i)));
+ }
+
+ return std::shared_ptr<Schema>(new Schema(fields));
+}
+
+} // namespace parquet
+
+} // namespace arrow

cpp/src/arrow/parquet/schema.h

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PARQUET_SCHEMA_H
+#define ARROW_PARQUET_SCHEMA_H
+
+#include <arrow/schema.h>
+#include <arrow/type.h>
+#include <parquet/schema/descriptor.h>
+#include <parquet/schema/types.h>
+
+#include <memory>
+
+namespace arrow {
+
+namespace parquet {
+
+std::shared_ptr<Field> NodeToField(const parquet_cpp::schema::NodePtr& node);
+std::shared_ptr<Schema> FromParquetSchema(
+ const parquet_cpp::SchemaDescriptor* parquet_schema);
+
+} // namespace parquet
+
+} // namespace arrow
+
+#endif

cpp/src/arrow/types/decimal.cc

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/types/decimal.h"
+
+#include <sstream>
+#include <string>
+
+namespace arrow {
+
+std::string DecimalType::ToString() const {
+ std::stringstream s;
+ s << "decimal(" << precision << ", " << scale << ")";
+ return s.str();
+}
+
+} // namespace arrow
+

cpp/src/arrow/types/decimal.h

@@ -18,13 +18,24 @@
 #ifndef ARROW_TYPES_DECIMAL_H
 #define ARROW_TYPES_DECIMAL_H
 
+#include <string>
+
 #include "arrow/type.h"
 
 namespace arrow {
 
 struct DecimalType : public DataType {
+ explicit DecimalType(int precision_, int scale_)
+ : DataType(Type::DECIMAL), precision(precision_),
+ scale(scale_) { }
  int precision;
  int scale;
+
+ static char const *name() {
+ return "decimal";
+ }
+
+ std::string ToString() const override;
 };
 
 } // namespace arrow