MarshalJSON: error:json: error calling MarshalJSON for type trap.object: field mi: json: unsupported value: encountered a cycle via []filedesc.Message

[xhd2015]

This may be caused by protobuf. But cyclic reference should also be handled normally.

[xhd2015]

So far we have tried to decycle the incoming data, but seems have big impact on memory.

The code is:

package trace_marshal

import (
	"reflect"
	"unsafe"
)

// NOTE: this file is a temporiray backup
// decyclic seems has signaficant memory consumption
// making it slow to decyclic when encounters large data
type decyclicer struct {
	seen map[uintptr]struct{}
}

func Decyclic(v interface{}) interface{} {
	if v == nil {
		return nil
	}
	d := &decyclicer{
		seen: map[uintptr]struct{}{},
	}
	d.clear(reflect.ValueOf(v), func(r reflect.Value) {
		v = r.Interface()
	})
	return v
}

func makeAddrable(v reflect.Value, set func(r reflect.Value)) reflect.Value {
	if v.CanAddr() {
		return v
	}
	p := reflect.New(v.Type())
	p.Elem().Set(v)
	x := p.Elem()
	set(x)
	return x
}

func (c *decyclicer) clear(v reflect.Value, set func(r reflect.Value)) {
	// fmt.Printf("clear: %v\n", v.Type())
	switch v.Kind() {
	case reflect.Ptr:
		if v.IsNil() {
			return
		}

		// only pointer can create cyclic
		ptr := v.Pointer()
		if ptr == 0 {
			return
		}
		if _, ok := c.seen[ptr]; ok {
			// fmt.Printf("found : 0x%x -> %v\n", ptr, v.Interface())
			set(reflect.Zero(v.Type()))
			return
		}
		c.seen[ptr] = struct{}{}
		defer delete(c.seen, ptr)

		v = makeAddrable(v, set)
		c.clear(v.Elem(), func(r reflect.Value) {
			v.Elem().Set(r)
		})
	case reflect.Interface:
		if v.IsNil() {
			return
		}
		v = makeAddrable(v, set)
		c.clear(v.Elem(), func(r reflect.Value) {
			// NOTE: interface{} is special
			// we can directly can call v.Set
			// instead of v.Elem().Set()
			v.Set(r)
			if v.Elem().Kind() == reflect.Ptr && v.Elem().IsNil() {
				// fmt.Printf("found isNil\n")
				// avoid {nil-value,non-nil type}
				set(reflect.Zero(v.Type()))
			}
		})
	case reflect.Array, reflect.Slice:
		switch v.Type().Elem().Kind() {
		case reflect.Int64, reflect.Int, reflect.Int32, reflect.Int16,
			reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16,
			reflect.Float64, reflect.Float32,
			reflect.String,
			reflect.Bool:
			return
		case reflect.Int8, reflect.Uint8:
			// []byte -> Uint8
			// ignore some: 10K JSON
			n := v.Len()
			if v.Kind() == reflect.Slice && n > 10*1024 {
				// reserve first 16 and last 16
				//   S[:16] ... + S[len(S)-16:]
				const reserve = 16
				const ellipse = 3
				const totalLen = reserve*2 + ellipse
				newSlice := reflect.MakeSlice(v.Type(), totalLen, totalLen)
				for i := 0; i < reserve; i++ {
					newSlice.Index(i).Set(v.Index(i))
				}
				for i := 0; i < ellipse; i++ {
					if v.Kind() == reflect.Uint8 {
						newSlice.Index(reserve + i).SetUint('.')
					} else if v.Kind() == reflect.Int8 {
						newSlice.Index(reserve + i).SetInt('.')
					}
				}
				for i := 0; i < reserve; i++ {
					newSlice.Index(reserve + ellipse + i).Set(v.Index(n - reserve + i))
				}
				set(newSlice)
			}
			return
		}
		v = makeAddrable(v, set)
		for i := 0; i < v.Len(); i++ {
			e := v.Index(i)
			c.clear(e, func(r reflect.Value) {
				e.Set(r)
			})
		}
	case reflect.Map:
		v = makeAddrable(v, set)
		iter := v.MapRange()
		// sets := [][2]reflect.Value{}
		for iter.Next() {
			vi := v.MapIndex(iter.Key())
			c.clear(vi, func(r reflect.Value) {
				v.SetMapIndex(iter.Key(), r)
			})
		}
	case reflect.Struct:
		// fmt.Printf("struct \n")
		// make struct addrable
		v = makeAddrable(v, set)

		for i := 0; i < v.NumField(); i++ {
			field := v.Field(i)
			if field.CanSet() {
				c.clear(field, func(r reflect.Value) {
					field.Set(r)
				})
			} else {
				e := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr()))
				c.clear(e.Elem(), func(r reflect.Value) {
					e.Elem().Set(r)
				})
				// panic(fmt.Errorf("cannot set: %v", field))
			}
		}
	case reflect.Chan, reflect.Func:
		// ignore
	default:
		// int
	}
}

[xhd2015]

Solution to the cyclic stack trace problem: separate stack trace hierarchical info from request response data.
Storing them into three sub keys: {trace, func, data}.

Example:

{
   "schema":"id_mapping",
   "trace":{"funcID":1, "dataID":1, "children":[]},
   "func":{ "<id>":{}},
   "data":{  "<id>":{} }
}

The data part can be individually marshaled, upon error of a specific object, it's data can be ignored, and show an error prompt in the box.

[xhd2015]

Few possible optimizations:

• error silent when marshaling results and objects (done)
• set a size limit, for example 4K, which will make marshaled data shrinked when exceeding that limit (done)
• limit the depth of the stack
• limit the appearance of a specific function

[xhd2015]

This is an example trace, which is the count of calls for each function inside each package. It is large enough to cause the browser tab to crash (original file size 116M):
stats.json

Highlight:

{
    "encoding/json": {
        "newTypeEncoder": 72
    },
    "some.git.com/slc/c/framework_routinelocal/src/impl": {
        "(*goroutineLocalImpl).Put": 31,
        "(*goroutineLocalImpl).Value": 103
    },
    "some.git.com/slc/c/go_tls": {
        "(*dataImpl).Value": 83,
        "Get": 103,
        "MakeData": 31,
        "Set": 31,
        "fetchDataMap": 134,
        "getTlsData": 134
    },
    "some.git.com/slc/c/go_tls/g": {
        "G": 134
    },
    "some.git.com/slc/upppb": {
        "(*BizCommon).ProtoReflect": 77
    },
    "some.git.com/slc/upppc": {
        "(*BizResult).ProtoReflect": 90
    },
    "github.com/golang/protobuf/proto": {
        "String": 36
    },
    "github.com/prometheus/client_model/go": {
        "(*LabelPair).GetName": 84
    },
    "google.golang.org/protobuf/encoding/protowire": {
        "ConsumeBytes": 9927,
        "ConsumeTag": 17035,
        "ConsumeVarint": 34070,
        "DecodeTag": 17035,
        "EncodeTag": 68,
        "SizeVarint": 68
    },
    "google.golang.org/protobuf/internal/filedesc": {
        "(*Base).FullName": 3086,
        "(*Base).Parent": 937,
        "(*Base).Syntax": 61,
        "(*Builder).optionsUnmarshaler": 3617,
        "(*Enum).unmarshalFull": 112,
        "(*EnumValue).unmarshalFull": 937,
        "(*Field).Cardinality": 708,
        "(*Field).ContainingOneof": 397,
        "(*Field).HasPresence": 104,
        "(*Field).IsList": 285,
        "(*Field).IsMap": 651,
        "(*Field).IsWeak": 139,
        "(*Field).Kind": 479,
        "(*Field).Message": 1201,
        "(*Field).Number": 722,
        "(*Field).unmarshalFull": 2051,
        "(*Fields).Get": 622,
        "(*Fields).Len": 684,
        "(*File).Syntax": 61,
        "(*File).lazyInit": 640,
        "(*File).resolveMessageDependency": 615,
        "(*Message).Fields": 582,
        "(*Message).lazyInit": 638,
        "(*Message).unmarshalFull": 416,
        "(*stringName).InitJSON": 2051,
        "appendFullName": 3084,
        "makeFullName": 625
    },
    "google.golang.org/protobuf/internal/filetype": {
        "(*resolverByIndex).FindMessageByIndex": 613,
        "depIdxs.Get": 622
    },
    "google.golang.org/protobuf/internal/impl": {
        "Export.protoMessageV2Of": 210,
        "offsetOf": 148,
        "pointer.Apply": 68,
        "pointer.IsNil": 405,
        "pointerOfIface": 265
    },
    "google.golang.org/protobuf/internal/strs": {
        "(*Builder).AppendFullName": 3084,
        "(*Builder).MakeString": 2683,
        "(*Builder).grow": 5767,
        "(*Builder).last": 5767,
        "EnforceUTF8": 14,
        "UnsafeString": 8851
    },
    "google.golang.org/protobuf/reflect/protoreflect": {
        "Value.IsValid": 2051
    }
}

We can see there are too much unrelated function calls. Thus we can filter out items with appearance more than 100 times by default

[xhd2015]

After applying the appearance limited shrinking, the size of the original JSON reduced from 116M to 7.3M, and the browser now can handle it.

And it seems the majority of the stack trace is kept. No significant information is lost.