[#2039] Support default value semantics: API changes (#63)

Author: Shenoda Guirguis

api/src/main/java/org/apache/iceberg/types/Types.java

@@ -20,6 +20,7 @@
 package org.apache.iceberg.types;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -414,42 +415,108 @@ public int hashCode() {
 
   public static class NestedField implements Serializable {
     public static NestedField optional(int id, String name, Type type) {
-      return new NestedField(true, id, name, type, null);
+      return new NestedField(true, id, name, type, null, null);
     }
 
     public static NestedField optional(int id, String name, Type type, String doc) {
-      return new NestedField(true, id, name, type, doc);
+      return new NestedField(true, id, name, type, null, doc);
+    }
+
+    public static NestedField optional(int id, String name, Type type, Object defaultValue, String doc) {
+      return new NestedField(true, id, name, type, defaultValue, doc);
     }
 
     public static NestedField required(int id, String name, Type type) {
-      return new NestedField(false, id, name, type, null);
+      return new NestedField(false, id, name, type, null, null);
     }
 
     public static NestedField required(int id, String name, Type type, String doc) {
-      return new NestedField(false, id, name, type, doc);
+      return new NestedField(false, id, name, type, null, doc);
+    }
+
+    public static NestedField required(int id, String name, Type type, Object defaultValue, String doc) {
+      validateDefaultValueForRequiredField(defaultValue, name);
+      return new NestedField(false, id, name, type, defaultValue, doc);
     }
 
     public static NestedField of(int id, boolean isOptional, String name, Type type) {
-      return new NestedField(isOptional, id, name, type, null);
+      return new NestedField(isOptional, id, name, type, null, null);
     }
 
     public static NestedField of(int id, boolean isOptional, String name, Type type, String doc) {
-      return new NestedField(isOptional, id, name, type, doc);
+      return new NestedField(isOptional, id, name, type, null, doc);
+    }
+
+    public static NestedField of(int id, boolean isOptional, String name, Type type, Object defaultValue, String doc) {
+      return new NestedField(isOptional, id, name, type, defaultValue, doc);
+    }
+
+    private static void validateDefaultValueForRequiredField(Object defaultValue, String fieldName) {
+      Preconditions.checkArgument(defaultValue != null,
+          "Cannot create NestedField with a null default for the required field: " +  fieldName);
+    }
+
+    private static void validateDefaultValue(Object defaultValue, Type type) {
+      if (defaultValue == null) {
+        return;
+      }
+      switch (type.typeId()) {
+        case STRUCT:
+          Preconditions.checkArgument(List.class.isInstance(defaultValue),
+              "defaultValue should be a List of Objects for StructType");
+          if (defaultValue == null) {
+            return;
+          }
+          List<Object> defaultList = (List) defaultValue;
+          Preconditions.checkArgument(defaultList.size() == type.asStructType().fields().size());
+          for (int i = 0; i < defaultList.size(); i++) {
+            NestedField.validateDefaultValue(defaultList.get(i), type.asStructType().fields().get(i).type);
+          }
+          break;
+        case LIST:
+          Preconditions.checkArgument(defaultValue instanceof ArrayList,
+              "defaultValue should be an ArrayList of Objects, for ListType");
+          List<Object> defaultArrayList = (ArrayList<Object>) defaultValue;
+          if (defaultArrayList == null || defaultArrayList.size() == 0) {
+            return;
+          }
+          defaultArrayList.forEach(dv -> NestedField.validateDefaultValue(dv, type.asListType().elementField.type));
+          break;
+        case MAP:
+          Preconditions.checkArgument(Map.class.isInstance(defaultValue),
+              "defaultValue should be an instance of Map for MapType");
+          Map<Object, Object> defaultMap = (Map<Object, Object>) defaultValue;
+          if (defaultMap == null || defaultMap.isEmpty()) {
+            return;
+          }
+          for (Map.Entry e : defaultMap.entrySet()) {
+            NestedField.validateDefaultValue(e.getKey(), type.asMapType().keyField.type);
+            NestedField.validateDefaultValue(e.getValue(), type.asMapType().valueField.type);
+          }
+          break;
+        default:
+          Preconditions.checkArgument(type.typeId().javaClass().isInstance(defaultValue),
+              "defaultValue should be of same java class of the type, defaultValue class: " + defaultValue.getClass() +
+              ", type class: " + type.typeId().javaClass());
+      }
     }
 
     private final boolean isOptional;
     private final int id;
     private final String name;
     private final Type type;
+    private final Object defaultValue;
     private final String doc;
 
-    private NestedField(boolean isOptional, int id, String name, Type type, String doc) {
+    private NestedField(boolean isOptional, int id, String name, Type type, Object defaultValue, String doc) {
       Preconditions.checkNotNull(name, "Name cannot be null");
       Preconditions.checkNotNull(type, "Type cannot be null");
+      validateDefaultValue(defaultValue, type);
       this.isOptional = isOptional;
       this.id = id;
       this.name = name;
       this.type = type;
+      this.defaultValue = defaultValue;
       this.doc = doc;
     }
 
@@ -461,7 +528,7 @@ public NestedField asOptional() {
       if (isOptional) {
         return this;
       }
-      return new NestedField(true, id, name, type, doc);
+      return new NestedField(true, id, name, type, defaultValue, doc);
     }
 
     public boolean isRequired() {
@@ -472,7 +539,15 @@ public NestedField asRequired() {
       if (!isOptional) {
         return this;
       }
-      return new NestedField(false, id, name, type, doc);
+      return new NestedField(false, id, name, type, defaultValue, doc);
+    }
+
+    public boolean hasDefaultValue() {
+      return defaultValue != null;
+    }
+
+    public Object getDefaultValue() {
+      return defaultValue;
     }
 
     public int fieldId() {
@@ -495,6 +570,7 @@ public String doc() {
     public String toString() {
       return String.format("%d: %s: %s %s",
           id, name, isOptional ? "optional" : "required", type) +
+          (hasDefaultValue() ? ", default value: " + defaultValue + ", " : "") +
           (doc != null ? " (" + doc + ")" : "");
     }
 
@@ -513,6 +589,8 @@ public boolean equals(Object o) {
         return false;
       } else if (!name.equals(that.name)) {
         return false;
+      } else if (!Objects.equals(defaultValue, that.defaultValue)) {
+        return false;
       } else if (!Objects.equals(doc, that.doc)) {
         return false;
       }
@@ -521,7 +599,8 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-      return Objects.hash(NestedField.class, id, isOptional, name, type);
+      return hasDefaultValue() ? Objects.hash(NestedField.class, id, isOptional, name, type, defaultValue) :
+          Objects.hash(NestedField.class, id, isOptional, name, type);
     }
   }
 
@@ -739,7 +818,6 @@ public boolean equals(Object o) {
       } else if (!(o instanceof ListType)) {
         return false;
       }
-
       ListType listType = (ListType) o;
       return elementField.equals(listType.elementField);
     }

api/src/test/java/org/apache/iceberg/types/TestDefaultValuesForContainerTypes.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.types;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.iceberg.types.Types.NestedField;
+import static org.apache.iceberg.types.Types.StructType;
+
+public class TestDefaultValuesForContainerTypes {
+
+  static NestedField intFieldType;
+  static NestedField stringFieldType;
+  static StructType structType;
+
+  @BeforeClass
+  public static void beforeClass() {
+    intFieldType = NestedField.optional(0, "optionalIntField", Types.IntegerType.get());
+    stringFieldType = NestedField.required(1, "requiredStringField", Types.StringType.get());
+    structType = StructType.of(Arrays.asList(intFieldType, stringFieldType));
+  }
+
+  @Test
+  public void testStructTypeDefault() {
+    List<Object> structDefaultvalue = new ArrayList<>();
+    structDefaultvalue.add(Integer.valueOf(1));
+    structDefaultvalue.add("two");
+    NestedField structField = NestedField.optional(2, "optionalStructField", structType, structDefaultvalue, "doc");
+    Assert.assertTrue(structField.hasDefaultValue());
+    Assert.assertEquals(structDefaultvalue, structField.getDefaultValue());
+  }
+
+  @Test (expected = IllegalArgumentException.class)
+  public void testStructTypeDefaultInvalidFieldsTypes() {
+    List<Object> structDefaultvalue = new ArrayList<>();
+    structDefaultvalue.add("one");
+    structDefaultvalue.add("two");
+    NestedField.optional(2, "optionalStructField", structType, structDefaultvalue, "doc");
+  }
+
+  @Test (expected = IllegalArgumentException.class)
+  public void testStructTypeDefaultInvalidNumberFields() {
+    List<Object> structDefaultvalue = new ArrayList<>();
+    structDefaultvalue.add(Integer.valueOf(1));
+    structDefaultvalue.add("two");
+    structDefaultvalue.add("three");
+    NestedField.optional(2, "optionalStructField", structType, structDefaultvalue, "doc");
+  }
+}

api/src/test/java/org/apache/iceberg/types/TestNestedFieldDefaultValues.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.types;
+
+import org.apache.iceberg.types.Types.NestedField;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+
+
+public class TestNestedFieldDefaultValues {
+
+  private final int id = 1;
+  private final String fieldName = "fieldName";
+  private final Type fieldType = Types.IntegerType.get();
+  private final String doc = "field doc";
+  private final Integer defaultValue = 100;
+
+  @Test
+  public void testConstructorsValidCases() {
+    // optional constructors
+    Assert.assertFalse(optional(id, fieldName, fieldType).hasDefaultValue());
+    Assert.assertFalse(optional(id, fieldName, fieldType, doc).hasDefaultValue());
+    NestedField nestedFieldWithDefault = optional(id, fieldName, fieldType, defaultValue, doc);
+    Assert.assertTrue(nestedFieldWithDefault.hasDefaultValue());
+    Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
+    nestedFieldWithDefault = optional(id, fieldName, fieldType, defaultValue, null);
+    Assert.assertTrue(nestedFieldWithDefault.hasDefaultValue());
+    Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
+
+    // required constructors
+    Assert.assertFalse(required(id, fieldName, fieldType).hasDefaultValue());
+    Assert.assertFalse(required(id, fieldName, fieldType, doc).hasDefaultValue());
+    nestedFieldWithDefault = required(id, fieldName, fieldType, defaultValue, doc);
+    Assert.assertTrue(nestedFieldWithDefault.hasDefaultValue());
+    Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
+    nestedFieldWithDefault = required(id, fieldName, fieldType, defaultValue, null);
+    Assert.assertTrue(nestedFieldWithDefault.hasDefaultValue());
+    Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
+
+    // of constructors
+    Assert.assertFalse(NestedField.of(id, true, fieldName, fieldType).hasDefaultValue());
+    Assert.assertFalse(NestedField.of(id, true, fieldName, fieldType, doc).hasDefaultValue());
+    nestedFieldWithDefault = NestedField.of(id, true, fieldName, fieldType, defaultValue, doc);
+    Assert.assertTrue(nestedFieldWithDefault.hasDefaultValue());
+    Assert.assertEquals(defaultValue, nestedFieldWithDefault.getDefaultValue());
+  }
+
+  @Test (expected =  IllegalArgumentException.class)
+  public void testRequiredNullDefault() {
+    // illegal case (required with null defaultValue)
+    required(id, fieldName, fieldType, null, doc);
+  }
+
+  @Test (expected = IllegalArgumentException.class)
+  public void testRequiredWithDefaultNullDefault() {
+    // illegal case (required with null defaultValue)
+    required(id, fieldName, fieldType, null, null);
+  }
+
+  @Test (expected = IllegalArgumentException.class)
+  public void testOptionalWithInvalidDefaultValueClass() {
+    // class of default value does not match class of type
+    Long wrongClassDefaultValue = 100L;
+    optional(id, fieldName, fieldType, wrongClassDefaultValue, doc);
+  }
+}

site/docs/spec.md

@@ -90,6 +90,8 @@ A table's **schema** is a list of named columns. All data types are either primi
 
 For the representations of these types in Avro, ORC, and Parquet file formats, see Appendix A.
 
+Default values for fields are supported, see Neted Types below.
+
 #### Nested Types
 
 A **`struct`** is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. Fields may have an optional comment or doc string.
@@ -98,6 +100,13 @@ A **`list`** is a collection of values with some element type. The element field
 
 A **`map`** is a collection of key-value pairs with a key type and a value type. Both the key field and value field each have an integer id that is unique in the table schema. Map keys are required and map values can be either optional or required. Both map keys and map values may be any type, including nested types.
 
+Iceberg supports default-value semantics for fields of nested types (i.e., struct, list and map). Specifically, a field 
+of a nested type field can have a default value that will be returned upon reading this field, if it is not manifested. 
+The default value can be defined with both required and optional fields. Null default values are allowed with optional
+fields only, and it's behavior is identical to optional fields with no default value, that is a Null is returned upon
+reading this field when it is not manifested.
+
+
 #### Primitive Types
 
 | Primitive type     | Description                                                              | Requirements                                     |
@@ -692,7 +701,6 @@ This serialization scheme is for storing single values as individual binary valu
 | **`list`**                   | Not supported                                                                                                |
 | **`map`**                    | Not supported                                                                                                |
 
-
 ## Format version changes
 
 ### Version 2