feat(fill_struct): Json and yaml deserialization rewrite

Merge pull request #699 from qri-io/fill_struct

base/fill_struct.go

@@ -0,0 +1,240 @@
+package base
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"time"
+)
+
+// FillStruct fills in the values of an arbitrary structure using an already deserialized
+// map of nested data. Fields names are case-insensitive. Unknown fields are treated as an
+// error, *unless* the output structure implementes the ArbitrarySetter interface.
+func FillStruct(fields map[string]interface{}, output interface{}) error {
+	target := reflect.ValueOf(output)
+	if target.Kind() == reflect.Ptr {
+		target = target.Elem()
+	}
+	return putFieldsToTargetStruct(fields, target)
+}
+
+// ArbitrarySetter should be implemented by structs that can store arbitrary fields in a private map.
+type ArbitrarySetter interface {
+	SetArbitrary(string, interface{}) error
+}
+
+// timeObj and ifaceObj are used for reflect.TypeOf
+var timeObj time.Time
+var ifaceObj interface{}
+
+// putFieldsToTargetStruct iterates over the fields in the target struct, and assigns each
+// field the value from the `fields` map. Recursively call this for an sub structures. Field
+// names are treated as case-insensitive. Return any errors found during this process, or nil if
+// there are no errors.
+func putFieldsToTargetStruct(fields map[string]interface{}, target reflect.Value) error {
+	if target.Kind() != reflect.Struct {
+		return fmt.Errorf("can only put fields to a struct")
+	}
+
+	// Collect errors that occur in this single call, at the end of the function, join them
+	// using newlines, if any exist.
+	errs := make([]string, 0)
+
+	// Collect real key names used by the `fields` map.
+	realKeys := make([]string, 0)
+	for k := range fields {
+		realKeys = append(realKeys, k)
+	}
+	// Handle case-insensitivity by building a map from lowercase keys to real keys.
+	caseMap := make(map[string]string)
+	for i := 0; i < len(realKeys); i++ {
+		realKey := realKeys[i]
+		lowerKey := strings.ToLower(realKey)
+		caseMap[lowerKey] = realKey
+	}
+
+	// Keep track of which keys have been used from the `fields` map
+	usedKeys := make(map[string]bool)
+
+	for i := 0; i < target.NumField(); i++ {
+		// Lowercase the key in order to make matching case-insensitive.
+		fieldName := target.Type().Field(i).Name
+		lowerName := strings.ToLower(fieldName)
+
+		val := fields[caseMap[lowerName]]
+		if val == nil {
+			// Nothing to assign to this field, go to next.
+			continue
+		}
+		usedKeys[caseMap[lowerName]] = true
+
+		err := putValueToPlace(val, target.Field(i))
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("field %s: %s", fieldName, err.Error()))
+		}
+	}
+
+	// If the target struct is able, assign unknown keys to it.
+	arbitrarySetter := getArbitrarySetter(target)
+
+	// Iterate over keys in the `fields` data, see if there were any keys that were not stored in
+	// the target struct.
+	for i := 0; i < len(realKeys); i++ {
+		k := realKeys[i]
+		if _, ok := usedKeys[k]; !ok {
+			// If target struct allows storing unknown keys to a map of arbitrary data.
+			if arbitrarySetter != nil {
+				arbitrarySetter.SetArbitrary(k, fields[k])
+				continue
+			}
+			// Otherwise, unknown fields are an error.
+			errs = append(errs, fmt.Sprintf("field \"%s\" not found in target", k))
+		}
+	}
+
+	if len(errs) == 0 {
+		return nil
+	}
+	return fmt.Errorf("%s", strings.Join(errs, "\n"))
+}
+
+func putValueToPlace(val interface{}, place reflect.Value) error {
+	switch place.Kind() {
+	case reflect.Int:
+		num, ok := val.(int)
+		if ok {
+			place.SetInt(int64(num))
+			return nil
+		}
+		numFloat, ok := val.(float64)
+		if ok {
+			place.SetInt(int64(numFloat))
+			return nil
+		}
+		return fmt.Errorf("need type int, value %s", val)
+	case reflect.Float64:
+		num, ok := val.(int)
+		if ok {
+			place.SetFloat(float64(num))
+			return nil
+		}
+		numFloat, ok := val.(float64)
+		if ok {
+			place.SetFloat(numFloat)
+			return nil
+		}
+		return fmt.Errorf("need type string, value %s", val)
+	case reflect.String:
+		text, ok := val.(string)
+		if ok {
+			place.SetString(text)
+			return nil
+		}
+		return fmt.Errorf("need type string, value %s", val)
+	case reflect.Bool:
+		b, ok := val.(bool)
+		if ok {
+			place.SetBool(b)
+			return nil
+		}
+		return fmt.Errorf("need type string, value %s", val)
+	case reflect.Struct:
+		// Specially handle time.Time, represented as a string, which needs to be parsed.
+		if place.Type() == reflect.TypeOf(timeObj) {
+			timeText, ok := val.(string)
+			if ok {
+				ts, err := time.Parse(time.RFC3339, timeText)
+				if err == nil {
+					place.Set(reflect.ValueOf(ts))
+					return nil
+				}
+				return err
+			}
+			return fmt.Errorf("need type time, value %s", val)
+		}
+		// Other struct types are not handled currently. Should probably do the same thing
+		// as what's done for `pointer` below.
+		return fmt.Errorf("unknown struct %s", place.Type())
+	case reflect.Map:
+		m, ok := val.(map[string]interface{})
+		if ok {
+			place.Set(reflect.ValueOf(m))
+			return nil
+		}
+		return fmt.Errorf("need type map, value %s", val)
+	case reflect.Slice:
+		slice, ok := val.([]interface{})
+		if !ok {
+			return fmt.Errorf("need type slice, value %s", val)
+		}
+		// Get size of type of the slice to deserialize.
+		size := len(slice)
+		sliceType := reflect.TypeOf(ifaceObj)
+		if size > 0 {
+			sliceType = place.Type().Elem()
+		}
+		// Construct a new, empty slice of the same size.
+		create := reflect.MakeSlice(reflect.SliceOf(sliceType), size, size)
+		// Fill in each element.
+		for i := 0; i < size; i++ {
+			elem := reflect.Indirect(reflect.New(sliceType))
+			err := putValueToPlace(slice[i], elem)
+			if err != nil {
+				return err
+			}
+			create.Index(i).Set(elem)
+		}
+		place.Set(create)
+		return nil
+	case reflect.Ptr:
+		// Allocate a new pointer for the sub-component to be filled in.
+		alloc := reflect.New(place.Type().Elem())
+		place.Set(alloc)
+		inner := alloc.Elem()
+		// For now, can only point to a struct.
+		if inner.Kind() != reflect.Struct {
+			return fmt.Errorf("can only assign to *struct")
+		}
+		// Struct must be assigned from a map.
+		component, err := toStringMap(val)
+		if err != nil {
+			return err
+		}
+		// Recursion to handle sub-component.
+		return putFieldsToTargetStruct(component, inner)
+	default:
+		return fmt.Errorf("unknown kind %s", place.Kind())
+	}
+}
+
+// toStringMap converts the input to a map[string] if able. This is needed because, while JSON
+// correctly deserializes sub structures to map[string], YAML instead deserializes to
+// map[interface{}]interface{}, so we need to manually convert this case to map[string].
+func toStringMap(val interface{}) (map[string]interface{}, error) {
+	m, ok := val.(map[string]interface{})
+	if ok {
+		return m, nil
+	}
+	imap, ok := val.(map[interface{}]interface{})
+	if ok {
+		convert := make(map[string]interface{})
+		for k, v := range imap {
+			convert[fmt.Sprintf("%v", k)] = v
+		}
+		return convert, nil
+	}
+	return nil, fmt.Errorf("could not convert to map[string]")
+}
+
+// getArbitrarySetter returns a ArbitrarySetter if the target implements it.
+func getArbitrarySetter(target reflect.Value) ArbitrarySetter {
+	if !target.CanAddr() {
+		return nil
+	}
+	ptr := target.Addr()
+	iface := ptr.Interface()
+	if setter, ok := iface.(ArbitrarySetter); ok {
+		return setter
+	}
+	return nil
+}