Move more parquet functionality behind experimental feature flag (#1032…

…) (#1134)

* Move more parquet functionality behind experimental feature flag (#1032)

* Fix logical conflicts

parquet/src/arrow/array_reader.rs

@@ -2469,8 +2469,8 @@ mod tests {
 
     #[test]
     fn test_complex_array_reader_dict_enc_string() {
-        use crate::encoding::{DictEncoder, Encoder};
-        use crate::memory::MemTracker;
+        use crate::encodings::encoding::{DictEncoder, Encoder};
+        use crate::util::memory::MemTracker;
         // Construct column schema
         let message_type = "
         message test_schema {

parquet/src/arrow/arrow_array_reader.rs

@@ -19,12 +19,12 @@ use super::array_reader::ArrayReader;
 use crate::arrow::schema::parquet_to_arrow_field;
 use crate::basic::Encoding;
 use crate::data_type::{ByteArray, ByteArrayType};
-use crate::decoding::{Decoder, DeltaByteArrayDecoder};
+use crate::encodings::decoding::{Decoder, DeltaByteArrayDecoder};
 use crate::errors::{ParquetError, Result};
 use crate::{
     column::page::{Page, PageIterator},
-    memory::ByteBufferPtr,
     schema::types::{ColumnDescPtr, ColumnDescriptor},
+    util::memory::ByteBufferPtr,
 };
 use arrow::{
     array::{ArrayRef, Int16Array},
@@ -1263,12 +1263,12 @@ mod tests {
     use crate::column::writer::ColumnWriter;
     use crate::data_type::ByteArray;
     use crate::data_type::ByteArrayType;
-    use crate::encoding::{DictEncoder, Encoder};
+    use crate::encodings::encoding::{DictEncoder, Encoder};
     use crate::file::properties::WriterProperties;
     use crate::file::reader::SerializedFileReader;
     use crate::file::serialized_reader::SliceableCursor;
     use crate::file::writer::{FileWriter, SerializedFileWriter, TryClone};
-    use crate::memory::MemTracker;
+    use crate::util::memory::MemTracker;
     use crate::schema::parser::parse_message_type;
     use crate::schema::types::SchemaDescriptor;
     use crate::util::test_common::page_util::{

parquet/src/arrow/arrow_writer.rs

@@ -26,7 +26,8 @@ use arrow_array::Array;
 
 use super::levels::LevelInfo;
 use super::schema::{
-    add_encoded_arrow_schema_to_metadata, decimal_length_from_precision,
+    add_encoded_arrow_schema_to_metadata, arrow_to_parquet_schema,
+    decimal_length_from_precision,
 };
 
 use crate::column::writer::ColumnWriter;
@@ -62,7 +63,7 @@ impl<W: 'static + ParquetWriter> ArrowWriter<W> {
         arrow_schema: SchemaRef,
         props: Option<WriterProperties>,
     ) -> Result<Self> {
-        let schema = crate::arrow::arrow_to_parquet_schema(&arrow_schema)?;
+        let schema = arrow_to_parquet_schema(&arrow_schema)?;
         // add serialized arrow schema
         let mut props = props.unwrap_or_else(|| WriterProperties::builder().build());
         add_encoded_arrow_schema_to_metadata(&arrow_schema, &mut props);
@@ -470,7 +471,7 @@ macro_rules! def_get_binary_array_fn {
     ($name:ident, $ty:ty) => {
         fn $name(array: &$ty) -> Vec<ByteArray> {
             let mut byte_array = ByteArray::new();
-            let ptr = crate::memory::ByteBufferPtr::new(
+            let ptr = crate::util::memory::ByteBufferPtr::new(
                 unsafe { array.value_data().typed_data::<u8>() }.to_vec(),
             );
             byte_array.set_data(ptr);

parquet/src/arrow/mod.rs

@@ -119,17 +119,19 @@
 //! ```
 
 experimental_mod!(array_reader);
-pub mod arrow_array_reader;
+experimental_mod!(arrow_array_reader);
 pub mod arrow_reader;
 pub mod arrow_writer;
-pub mod converter;
+experimental_mod!(converter);
 pub(in crate::arrow) mod levels;
 pub(in crate::arrow) mod record_reader;
-pub mod schema;
+experimental_mod!(schema);
 
 pub use self::arrow_reader::ArrowReader;
 pub use self::arrow_reader::ParquetFileArrowReader;
 pub use self::arrow_writer::ArrowWriter;
+
+#[cfg(feature = "experimental")]
 pub use self::schema::{
     arrow_to_parquet_schema, parquet_to_arrow_schema, parquet_to_arrow_schema_by_columns,
     parquet_to_arrow_schema_by_root_columns,

parquet/src/compression.rs

@@ -20,26 +20,30 @@
 //! See [`Compression`](crate::basic::Compression) enum for all available compression
 //! algorithms.
 //!
-//! # Example
-//!
-//! ```no_run
-//! use parquet::{basic::Compression, compression::create_codec};
-//!
-//! let mut codec = match create_codec(Compression::SNAPPY) {
-//!     Ok(Some(codec)) => codec,
-//!     _ => panic!(),
-//! };
-//!
-//! let data = vec![b'p', b'a', b'r', b'q', b'u', b'e', b't'];
-//! let mut compressed = vec![];
-//! codec.compress(&data[..], &mut compressed).unwrap();
-//!
-//! let mut output = vec![];
-//! codec.decompress(&compressed[..], &mut output).unwrap();
-//!
-//! assert_eq!(output, data);
-//! ```
-
+#[cfg_attr(
+    feature = "experimental",
+    doc = r##"
+# Example
+
+```no_run
+use parquet::{basic::Compression, compression::create_codec};
+
+let mut codec = match create_codec(Compression::SNAPPY) {
+ Ok(Some(codec)) => codec,
+ _ => panic!(),
+};
+
+let data = vec![b'p', b'a', b'r', b'q', b'u', b'e', b't'];
+let mut compressed = vec![];
+codec.compress(&data[..], &mut compressed).unwrap();
+
+let mut output = vec![];
+codec.decompress(&compressed[..], &mut output).unwrap();
+
+assert_eq!(output, data);
+```
+"##
+)]
 use crate::basic::Compression as CodecType;
 use crate::errors::{ParquetError, Result};
 

parquet/src/encodings/encoding.rs

@@ -922,7 +922,7 @@ mod tests {
 
     use std::sync::Arc;
 
-    use crate::decoding::{get_decoder, Decoder, DictDecoder, PlainDecoder};
+    use crate::encodings::decoding::{get_decoder, Decoder, DictDecoder, PlainDecoder};
     use crate::schema::types::{
         ColumnDescPtr, ColumnDescriptor, ColumnPath, Type as SchemaType,
     };

parquet/src/lib.rs

@@ -42,31 +42,36 @@
 ///
 /// Experimental modules have no stability guarantees
 macro_rules! experimental_mod {
-    ($module:ident) => {
+    ($module:ident $(, #[$meta:meta])*) => {
         #[cfg(feature = "experimental")]
         #[doc(hidden)]
+        $(#[$meta])*
         pub mod $module;
         #[cfg(not(feature = "experimental"))]
+        $(#[$meta])*
         mod $module;
     };
 }
 
 #[macro_use]
 pub mod errors;
 pub mod basic;
-#[macro_use]
-pub mod data_type;
+experimental_mod!(data_type, #[macro_use]);
 
 // Exported for external use, such as benchmarks
+#[cfg(feature = "experimental")]
+#[doc(hidden)]
 pub use self::encodings::{decoding, encoding};
+
+#[cfg(feature = "experimental")]
+#[doc(hidden)]
 pub use self::util::memory;
 
-#[macro_use]
-pub mod util;
+experimental_mod!(util, #[macro_use]);
 #[cfg(any(feature = "arrow", test))]
 pub mod arrow;
 pub mod column;
-pub mod compression;
+experimental_mod!(compression);
 mod encodings;
 pub mod file;
 pub mod record;