diff --git a/ast/encode_test.go b/ast/encode_test.go index a63e1f85f..022e717c2 100644 --- a/ast/encode_test.go +++ b/ast/encode_test.go @@ -17,215 +17,217 @@ package ast import ( - `encoding/json` - `runtime` - `sync` - `testing` + "encoding/json" + "runtime" + "sync" + "testing" - `github.com/bytedance/sonic/internal/native/types` - `github.com/stretchr/testify/assert` - `github.com/stretchr/testify/require` + "github.com/bytedance/sonic/internal/native/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGC_Encode(t *testing.T) { - if debugSyncGC { - return - } - root, err := NewSearcher(_TwitterJson).GetByPath() - if err != nil { - t.Fatal(err) - } - root.LoadAll() - _, err = root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - wg := &sync.WaitGroup{} - N := 10000 - for i:=0; i 10 { - t.Fatal() - } + var obj = map[string]interface{}{ + "a": json.RawMessage(" [} "), + } + buf := make([]byte, 0, 10) + _ = EncodeInto(&buf, obj, 0) + if len(buf) < 0 || len(buf) > 10 { + t.Fatal() + } } type MarshalerImpl struct { - X int + X int } func (self *MarshalerImpl) MarshalJSON() ([]byte, error) { - ret := []byte(strconv.Itoa(self.X)) - return append(ret, " "...), nil + ret := []byte(strconv.Itoa(self.X)) + return append(ret, " "...), nil } type MarshalerStruct struct { - V MarshalerImpl + V MarshalerImpl } type MarshalerErrorStruct struct { - V MarshalerImpl + V MarshalerImpl } func (self *MarshalerErrorStruct) MarshalJSON() ([]byte, error) { - return []byte(`[""] {`), nil + return []byte(`[""] {`), nil } type RawMessageStruct struct { - X json.RawMessage + X json.RawMessage } type TextMarshalerImpl struct { - X string + X string } func (self *TextMarshalerImpl) MarshalText() ([]byte, error) { - return []byte(self.X), nil + return []byte(self.X), nil } type TextMarshalerImplV struct { - X string + X string } func (self TextMarshalerImplV) MarshalText() ([]byte, error) { - return []byte(self.X), nil + return []byte(self.X), nil } type TextMarshalerStruct struct { - V TextMarshalerImpl + V TextMarshalerImpl } func TestTextMarshalTextKey_SortKeys(t *testing.T) { - v := map[*TextMarshalerImpl]string{ - {"b"}: "b", - {"c"}: "c", - {"a"}: "a", - } - ret, err := Encode(v, SortMapKeys) - require.NoError(t, err) - require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) - - v2 := map[TextMarshalerImplV]string{ - {"b"}: "b", - {"c"}: "c", - {"a"}: "a", - } - ret, err = Encode(v2, SortMapKeys) - require.NoError(t, err) - require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) - - v3 := map[encoding.TextMarshaler]string{ - TextMarshalerImplV{"b"}: "b", - &TextMarshalerImpl{"c"}: "c", - TextMarshalerImplV{"a"}: "a", - } - ret, err = Encode(v3, SortMapKeys) - require.NoError(t, err) - require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) + v := map[*TextMarshalerImpl]string{ + {"b"}: "b", + {"c"}: "c", + {"a"}: "a", + } + ret, err := Encode(v, SortMapKeys) + require.NoError(t, err) + require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) + + v2 := map[TextMarshalerImplV]string{ + {"b"}: "b", + {"c"}: "c", + {"a"}: "a", + } + ret, err = Encode(v2, SortMapKeys) + require.NoError(t, err) + require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) + + v3 := map[encoding.TextMarshaler]string{ + TextMarshalerImplV{"b"}: "b", + &TextMarshalerImpl{"c"}: "c", + TextMarshalerImplV{"a"}: "a", + } + ret, err = Encode(v3, SortMapKeys) + require.NoError(t, err) + require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret)) } func TestEncoder_EscapeHTML(t *testing.T) { - // test data from libfuzzer - test := []string{ - "&&&&&&&&&&&&&&&&&&&&&&&\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&", - "{\"\"\u2028\x94\xe2\x00\x00\x00\x00\x00\x00\x00\x00\u2028\x80\u2028\x80\u2028\xe2\u2028\x8a\u2028⑀\xa8\x8a\xa8\xe2\u2028\xe2\u2028\xe2\u2028\xe2\u2000\x8d\xe2\u2028\xe2\u2028\xe2\xe2\xa8\"}", - } - for _, s := range(test) { - data := []byte(s) - sdst := HTMLEscape(nil, data) - var dst bytes.Buffer - json.HTMLEscape(&dst, data) - require.Equal(t, string(sdst), dst.String()) - } + // test data from libfuzzer + test := []string{ + "&&&&&&&&&&&&&&&&&&&&&&&\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&", + "{\"\"\u2028\x94\xe2\x00\x00\x00\x00\x00\x00\x00\x00\u2028\x80\u2028\x80\u2028\xe2\u2028\x8a\u2028⑀\xa8\x8a\xa8\xe2\u2028\xe2\u2028\xe2\u2028\xe2\u2000\x8d\xe2\u2028\xe2\u2028\xe2\xe2\xa8\"}", + } + for _, s := range test { + data := []byte(s) + sdst := HTMLEscape(nil, data) + var dst bytes.Buffer + json.HTMLEscape(&dst, data) + require.Equal(t, string(sdst), dst.String()) + } } func TestEncoder_Marshal_EscapeHTML_LargeJson(t *testing.T) { - buf1, err1 := Encode(&_BindingValue, SortMapKeys | EscapeHTML) - require.NoError(t, err1) - buf2, err2 :=json.Marshal(&_BindingValue) - require.NoError(t, err2) - require.Equal(t, buf1, buf2) + buf1, err1 := Encode(&_BindingValue, SortMapKeys|EscapeHTML) + require.NoError(t, err1) + buf2, err2 := json.Marshal(&_BindingValue) + require.NoError(t, err2) + require.Equal(t, buf1, buf2) } var _GenericValue interface{} var _BindingValue TwitterStruct func init() { - _ = json.Unmarshal([]byte(TwitterJson), &_GenericValue) - _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) + _ = json.Unmarshal([]byte(TwitterJson), &_GenericValue) + _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) } func TestEncoder_Generic(t *testing.T) { - v, e := Encode(_GenericValue, 0) - require.NoError(t, e) - println(string(v)) + v, e := Encode(_GenericValue, 0) + require.NoError(t, e) + println(string(v)) } func TestEncoder_Binding(t *testing.T) { - v, e := Encode(_BindingValue, 0) - require.NoError(t, e) - println(string(v)) + v, e := Encode(_BindingValue, 0) + require.NoError(t, e) + println(string(v)) } func TestEncoder_MapSortKey(t *testing.T) { - m := map[string]string { - "C": "third", - "D": "forth", - "A": "first", - "F": "sixth", - "E": "fifth", - "B": "second", - } - v, e := Encode(m, SortMapKeys) - require.NoError(t, e) - require.Equal(t, `{"A":"first","B":"second","C":"third","D":"forth","E":"fifth","F":"sixth"}`, string(v)) + m := map[string]string{ + "C": "third", + "D": "forth", + "A": "first", + "F": "sixth", + "E": "fifth", + "B": "second", + } + v, e := Encode(m, SortMapKeys) + require.NoError(t, e) + require.Equal(t, `{"A":"first","B":"second","C":"third","D":"forth","E":"fifth","F":"sixth"}`, string(v)) } func BenchmarkEncoder_Generic_Sonic(b *testing.B) { - _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) - } + _, _ = Encode(_GenericValue, SortMapKeys|EscapeHTML|CompactMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = Encode(_GenericValue, SortMapKeys|EscapeHTML|CompactMarshaler) + } } func BenchmarkEncoder_Generic_Sonic_Fast(b *testing.B) { - _, _ = Encode(_GenericValue, 0) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = Encode(_GenericValue, 0) - } + _, _ = Encode(_GenericValue, 0) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = Encode(_GenericValue, 0) + } } func BenchmarkEncoder_Generic_StdLib(b *testing.B) { - _, _ = json.Marshal(_GenericValue) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = json.Marshal(_GenericValue) - } + _, _ = json.Marshal(_GenericValue) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = json.Marshal(_GenericValue) + } } func BenchmarkEncoder_Binding_Sonic(b *testing.B) { - _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) - } + _, _ = Encode(&_BindingValue, SortMapKeys|EscapeHTML|CompactMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = Encode(&_BindingValue, SortMapKeys|EscapeHTML|CompactMarshaler) + } } func BenchmarkEncoder_Binding_Sonic_Fast(b *testing.B) { - _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) - } + _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) + } } func BenchmarkEncoder_Binding_StdLib(b *testing.B) { - _, _ = json.Marshal(&_BindingValue) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = json.Marshal(&_BindingValue) - } + _, _ = json.Marshal(&_BindingValue) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = json.Marshal(&_BindingValue) + } } func BenchmarkEncoder_Parallel_Generic_Sonic(b *testing.B) { - _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler) - } - }) + _, _ = Encode(_GenericValue, SortMapKeys|EscapeHTML|CompactMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = Encode(_GenericValue, SortMapKeys|EscapeHTML|CompactMarshaler) + } + }) } func BenchmarkEncoder_Parallel_Generic_Sonic_Fast(b *testing.B) { - _, _ = Encode(_GenericValue, NoQuoteTextMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = Encode(_GenericValue, NoQuoteTextMarshaler) - } - }) + _, _ = Encode(_GenericValue, NoQuoteTextMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = Encode(_GenericValue, NoQuoteTextMarshaler) + } + }) } func BenchmarkEncoder_Parallel_Generic_StdLib(b *testing.B) { - _, _ = json.Marshal(_GenericValue) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = json.Marshal(_GenericValue) - } - }) + _, _ = json.Marshal(_GenericValue) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = json.Marshal(_GenericValue) + } + }) } func BenchmarkEncoder_Parallel_Binding_Sonic(b *testing.B) { - _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler) - } - }) + _, _ = Encode(&_BindingValue, SortMapKeys|EscapeHTML|CompactMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = Encode(&_BindingValue, SortMapKeys|EscapeHTML|CompactMarshaler) + } + }) } func BenchmarkEncoder_Parallel_Binding_Sonic_Fast(b *testing.B) { - _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) - } - }) + _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler) + } + }) } func BenchmarkEncoder_Parallel_Binding_StdLib(b *testing.B) { - _, _ = json.Marshal(&_BindingValue) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, _ = json.Marshal(&_BindingValue) - } - }) + _, _ = json.Marshal(&_BindingValue) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, _ = json.Marshal(&_BindingValue) + } + }) } func BenchmarkHTMLEscape_Sonic(b *testing.B) { - jsonByte := []byte(TwitterJson) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - var buf []byte - for i := 0; i < b.N; i++ { - buf = HTMLEscape(nil, jsonByte) - } - _ = buf + jsonByte := []byte(TwitterJson) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + var buf []byte + for i := 0; i < b.N; i++ { + buf = HTMLEscape(nil, jsonByte) + } + _ = buf } func BenchmarkHTMLEscape_StdLib(b *testing.B) { - jsonByte := []byte(TwitterJson) - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - var buf []byte - for i := 0; i < b.N; i++ { - out := bytes.NewBuffer(make([]byte, 0, len(TwitterJson) * 6 / 5)) - json.HTMLEscape(out, jsonByte) - buf = out.Bytes() - } - _ = buf + jsonByte := []byte(TwitterJson) + b.SetBytes(int64(len(TwitterJson))) + b.ResetTimer() + var buf []byte + for i := 0; i < b.N; i++ { + out := bytes.NewBuffer(make([]byte, 0, len(TwitterJson)*6/5)) + json.HTMLEscape(out, jsonByte) + buf = out.Bytes() + } + _ = buf } - func BenchmarkValidate_Sonic(b *testing.B) { - var data = rt.Str2Mem(TwitterJson) - ok, s := Valid(data) - if !ok { - b.Fatal(s) - } - b.SetBytes(int64(len(TwitterJson))) - b.ResetTimer() - for i:=0; i