feat: expose data file serialized (#26)

* expose data file serialized

* fix

Author: chenzl25

crates/iceberg/src/spec/manifest/_serde.rs

@@ -98,7 +98,8 @@ impl ManifestEntryV1 {
 
 #[serde_as]
 #[derive(Serialize, Deserialize)]
-pub(super) struct DataFileSerde {
+/// Data file
+pub struct DataFileSerde {
     #[serde(default)]
     content: i32,
     file_path: String,
@@ -126,6 +127,7 @@ pub(super) struct DataFileSerde {
 }
 
 impl DataFileSerde {
+    /// Try to convert a `super::DataFile` to a `DataFileSerde`.
     pub fn try_from(
         value: super::DataFile,
         partition_type: &StructType,
@@ -164,6 +166,7 @@ impl DataFileSerde {
         })
     }
 
+    /// Try to convert a `DataFileSerde` to a `super::DataFile`.
     pub fn try_into(
         self,
         partition_spec_id: i32,
@@ -236,7 +239,7 @@ impl DataFileSerde {
 }
 
 #[serde_as]
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 #[cfg_attr(test, derive(Debug, PartialEq, Eq))]
 struct BytesEntry {
     key: i32,
@@ -277,7 +280,7 @@ fn to_bytes_entry(v: impl IntoIterator<Item = (i32, Datum)>) -> Result<Vec<Bytes
     Ok(bs)
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 #[cfg_attr(test, derive(Debug, PartialEq, Eq))]
 struct I64Entry {
     key: i32,

crates/iceberg/src/spec/manifest/mod.rs

@@ -16,6 +16,8 @@
 // under the License.
 
 mod _serde;
+/// Data file
+pub use _serde::DataFileSerde as SerializedDataFile;
 
 mod data_file;
 pub use data_file::*;

crates/iceberg/src/spec/values.rs

@@ -2242,6 +2242,8 @@ mod timestamptz {
 }
 
 mod _serde {
+    use std::collections::HashMap;
+
     use serde::de::Visitor;
     use serde::ser::{SerializeMap, SerializeSeq, SerializeStruct};
     use serde::{Deserialize, Serialize};
@@ -2252,7 +2254,7 @@ mod _serde {
     use crate::spec::{MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, PrimitiveType, Type};
     use crate::{Error, ErrorKind};
 
-    #[derive(SerializeDerive, DeserializeDerive, Debug)]
+    #[derive(SerializeDerive, DeserializeDerive, Debug, Clone)]
     #[serde(transparent)]
     /// Raw literal representation used for serde. The serialize way is used for Avro serializer.
     pub struct RawLiteral(RawLiteralEnum);
@@ -2840,22 +2842,24 @@ mod _serde {
                     optional: _,
                 }) => match ty {
                     Type::Struct(struct_ty) => {
-                        let iters: Vec<Option<Literal>> = required
-                            .into_iter()
-                            .map(|(field_name, value)| {
-                                let field = struct_ty
-                                    .field_by_name(field_name.as_str())
-                                    .ok_or_else(|| {
-                                        invalid_err_with_reason(
-                                            "record",
-                                            &format!("field {} is not exist", &field_name),
-                                        )
-                                    })?;
-                                let value = value.try_into(&field.field_type)?;
-                                Ok(value)
+                        let mut value_map: HashMap<String, RawLiteralEnum> =
+                            required.into_iter().collect();
+                        let values = struct_ty
+                            .fields()
+                            .iter()
+                            .map(|f| {
+                                if let Some(raw_value) = value_map.remove(&f.name) {
+                                    let value = raw_value.try_into(&f.field_type)?;
+                                    Ok(value)
+                                } else {
+                                    Err(invalid_err_with_reason(
+                                        "record",
+                                        &format!("field {} is not exist", &f.name),
+                                    ))
+                                }
                             })
-                            .collect::<Result<_, Error>>()?;
-                        Ok(Some(Literal::Struct(super::Struct::from_iter(iters))))
+                            .collect::<Result<Vec<_>, Error>>()?;
+                        Ok(Some(Literal::Struct(super::Struct::from_iter(values))))
                     }
                     Type::Map(map_ty) => {
                         if *map_ty.key_field.field_type != Type::Primitive(PrimitiveType::String) {