Skip to content

Commit

Permalink
pull in a forked mergo: when we merge 2 structs, we want non-nil poin…
Browse files Browse the repository at this point in the history
…ters to false-like values (0, 0.0, false) to count as non-empty for merging

this lets us properly merge several json config structs
  • Loading branch information
SergeyTsalkov committed Dec 18, 2018
1 parent b10c56a commit 56b09c4
Show file tree
Hide file tree
Showing 9 changed files with 550 additions and 39 deletions.
23 changes: 23 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,29 @@ func (c *ConfigType) JobOptionsForQueue(queue string) (opts JobOptions) {
return
}

func (q *Queue) DeepJobOptions() (j JobOptions) {
j.Merge(q.JobOptions)
j.Merge(Config.GlobalJobOptions)
j.Merge(DefaultJobOptions)
return
}

func (q *Queue) PendingList() string {
return fmt.Sprintf("%s:queue:%s:pending", Config.ClusterName, q.Name)
}

func (q *Queue) DoneList() string {
return fmt.Sprintf("%s:queue:%s:done", Config.ClusterName, q.Name)
}

func (q *Queue) FailedList() string {
return fmt.Sprintf("%s:queue:%s:failed", Config.ClusterName, q.Name)
}

func (q *Queue) DelayedList() string {
return fmt.Sprintf("%s:queue:%s:delayed", Config.ClusterName, q.Name)
}

// this use of init sucks, but we'll have to fix every "var redisClient = myredis.Get()"
// that is in a header to avoid it -- let's do this later!
func init() {
Expand Down
22 changes: 3 additions & 19 deletions config/joboptions.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@ package config

import (
"log"
"reflect"
"time"

"github.com/imdario/mergo"
"brooce/mergo"
"github.com/mitchellh/copystructure"
)

Expand Down Expand Up @@ -59,7 +58,7 @@ func (j *JobOptions) MaxTries() int {
if j.MaxTries_ != nil && *j.MaxTries_ > 0 {
return *j.MaxTries_
}
return 3600
return 1
}

func (j *JobOptions) KillOnDelay() bool {
Expand Down Expand Up @@ -159,22 +158,7 @@ func (j *JobOptions) clone() JobOptions {
func (j *JobOptions) Merge(parent JobOptions) {
// don't want to copy pointers to values in parent -- we might change those values later, which would
// inadvertently change parent
if err := mergo.Merge(j, parent.clone(), mergo.WithTransformers(ptrTransformer{})); err != nil {
if err := mergo.Merge(j, parent.clone()); err != nil {
log.Fatalf("merge wtf: %+v", err)
}
}

// when merging, a non-nil *bool or *int should be treated as non-zero
type ptrTransformer struct{}

func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
if typ.Kind() == reflect.Ptr && (typ.Elem().Kind() == reflect.Bool || typ.Elem().Kind() == reflect.Int) {
return func(dst, src reflect.Value) error {
if dst.CanSet() && dst.IsNil() && !src.IsNil() {
dst.Set(src)
}
return nil
}
}
return nil
}
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ require (
github.com/go-redis/redis v6.14.1+incompatible
github.com/golang/protobuf v1.2.0 // indirect
github.com/hpcloud/tail v1.0.0 // indirect
github.com/imdario/mergo v0.3.6
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
github.com/mitchellh/copystructure v1.0.0
github.com/onsi/ginkgo v1.6.0 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
Expand Down
9 changes: 1 addition & 8 deletions listing/queues.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,7 @@ func (q *QueueInfoType) DelayedList() string {
return fmt.Sprintf("%s:queue:%s:delayed", config.Config.ClusterName, q.Name)
}

func (q *QueueInfoType) JobOptions() config.JobOptions {
opts := config.JobOptions{}
opts.Merge(config.Config.JobOptionsForQueue(q.Name))
opts.Merge(config.Config.GlobalJobOptions)
opts.Merge(config.DefaultJobOptions)
return opts
}

// global list of queues, including those on other machines in our cluster
func Queues(short bool) (queueHash map[string]*QueueInfoType, err error) {
var workers []*heartbeat.HeartbeatType
workers, err = RunningWorkers()
Expand Down
174 changes: 174 additions & 0 deletions mergo/map.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
// Copyright 2014 Dario Castañé. All rights reserved.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Based on src/pkg/reflect/deepequal.go from official
// golang's stdlib.

package mergo

import (
"fmt"
"reflect"
"unicode"
"unicode/utf8"
)

func changeInitialCase(s string, mapper func(rune) rune) string {
if s == "" {
return s
}
r, n := utf8.DecodeRuneInString(s)
return string(mapper(r)) + s[n:]
}

func isExported(field reflect.StructField) bool {
r, _ := utf8.DecodeRuneInString(field.Name)
return r >= 'A' && r <= 'Z'
}

// Traverses recursively both values, assigning src's fields values to dst.
// The map argument tracks comparisons that have already been seen, which allows
// short circuiting on recursive types.
func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) {
overwrite := config.Overwrite
if dst.CanAddr() {
addr := dst.UnsafeAddr()
h := 17 * addr
seen := visited[h]
typ := dst.Type()
for p := seen; p != nil; p = p.next {
if p.ptr == addr && p.typ == typ {
return nil
}
}
// Remember, remember...
visited[h] = &visit{addr, typ, seen}
}
zeroValue := reflect.Value{}
switch dst.Kind() {
case reflect.Map:
dstMap := dst.Interface().(map[string]interface{})
for i, n := 0, src.NumField(); i < n; i++ {
srcType := src.Type()
field := srcType.Field(i)
if !isExported(field) {
continue
}
fieldName := field.Name
fieldName = changeInitialCase(fieldName, unicode.ToLower)
if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) {
dstMap[fieldName] = src.Field(i).Interface()
}
}
case reflect.Ptr:
if dst.IsNil() {
v := reflect.New(dst.Type().Elem())
dst.Set(v)
}
dst = dst.Elem()
fallthrough
case reflect.Struct:
srcMap := src.Interface().(map[string]interface{})
for key := range srcMap {
srcValue := srcMap[key]
fieldName := changeInitialCase(key, unicode.ToUpper)
dstElement := dst.FieldByName(fieldName)
if dstElement == zeroValue {
// We discard it because the field doesn't exist.
continue
}
srcElement := reflect.ValueOf(srcValue)
dstKind := dstElement.Kind()
srcKind := srcElement.Kind()
if srcKind == reflect.Ptr && dstKind != reflect.Ptr {
srcElement = srcElement.Elem()
srcKind = reflect.TypeOf(srcElement.Interface()).Kind()
} else if dstKind == reflect.Ptr {
// Can this work? I guess it can't.
if srcKind != reflect.Ptr && srcElement.CanAddr() {
srcPtr := srcElement.Addr()
srcElement = reflect.ValueOf(srcPtr)
srcKind = reflect.Ptr
}
}

if !srcElement.IsValid() {
continue
}
if srcKind == dstKind {
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface {
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else if srcKind == reflect.Map {
if err = deepMap(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else {
return fmt.Errorf("type mismatch on %s field: found %v, expected %v", fieldName, srcKind, dstKind)
}
}
}
return
}

// Map sets fields' values in dst from src.
// src can be a map with string keys or a struct. dst must be the opposite:
// if src is a map, dst must be a valid pointer to struct. If src is a struct,
// dst must be map[string]interface{}.
// It won't merge unexported (private) fields and will do recursively
// any exported field.
// If dst is a map, keys will be src fields' names in lower camel case.
// Missing key in src that doesn't match a field in dst will be skipped. This
// doesn't apply if dst is a map.
// This is separated method from Merge because it is cleaner and it keeps sane
// semantics: merging equal types, mapping different (restricted) types.
func Map(dst, src interface{}, opts ...func(*Config)) error {
return _map(dst, src, opts...)
}

// MapWithOverwrite will do the same as Map except that non-empty dst attributes will be overridden by
// non-empty src attribute values.
// Deprecated: Use Map(…) with WithOverride
func MapWithOverwrite(dst, src interface{}, opts ...func(*Config)) error {
return _map(dst, src, append(opts, WithOverride)...)
}

func _map(dst, src interface{}, opts ...func(*Config)) error {
var (
vDst, vSrc reflect.Value
err error
)
config := &Config{}

for _, opt := range opts {
opt(config)
}

if vDst, vSrc, err = resolveValues(dst, src); err != nil {
return err
}
// To be friction-less, we redirect equal-type arguments
// to deepMerge. Only because arguments can be anything.
if vSrc.Kind() == vDst.Kind() {
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}
switch vSrc.Kind() {
case reflect.Struct:
if vDst.Kind() != reflect.Map {
return ErrExpectedMapAsDestination
}
case reflect.Map:
if vDst.Kind() != reflect.Struct {
return ErrExpectedStructAsDestination
}
default:
return ErrNotSupported
}
return deepMap(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}
Loading

0 comments on commit 56b09c4

Please sign in to comment.