From 1b56618730198932694766cd486daa03c3a0aa93 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 8 Feb 2023 14:57:49 +0800 Subject: [PATCH 1/3] feat: fallback api on unsupported go version --- README.md | 2 +- ast/api_amd64.go | 4 ++-- ast/api_compat.go | 3 +-- ast/stubs_go115.go | 2 +- compat.go | 4 +--- compat_test.go | 3 --- decode_float_test.go | 2 +- decode_test.go | 3 +-- encode_test.go | 2 +- search_test.go | 2 +- sonic.go | 2 +- use_number_test.go | 2 +- 12 files changed, 12 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2e90b2e36..9cc42c462 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A blazingly fast JSON serializing & deserializing library, accelerated by JIT (just-in-time compiling) and SIMD (single-instruction-multiple-data). ## Requirement -- Go 1.15/1.16/1.17/1.18/1.19 +- Go 1.15~1.20 - Linux/MacOS/Windows - Amd64 ARCH diff --git a/ast/api_amd64.go b/ast/api_amd64.go index 951c18d5b..6b3458aee 100644 --- a/ast/api_amd64.go +++ b/ast/api_amd64.go @@ -1,5 +1,5 @@ -//go:build amd64 -// +build amd64 +// +build amd64,go1.15,!go1.21 + package ast diff --git a/ast/api_compat.go b/ast/api_compat.go index 5a2e3fa4c..5eebf6802 100644 --- a/ast/api_compat.go +++ b/ast/api_compat.go @@ -1,5 +1,4 @@ -//go:build !amd64 -// +build !amd64 +// +build !amd64 !go1.15 go1.21 package ast diff --git a/ast/stubs_go115.go b/ast/stubs_go115.go index 579652426..37b9451f0 100644 --- a/ast/stubs_go115.go +++ b/ast/stubs_go115.go @@ -1,4 +1,4 @@ -// +build go1.15,!go1.20 +// +build !go1.20 /* * Copyright 2021 ByteDance Inc. diff --git a/compat.go b/compat.go index 11733ba43..e2f67c16d 100644 --- a/compat.go +++ b/compat.go @@ -1,5 +1,4 @@ -//go:build !amd64 -// +build !amd64 +// +build !amd64 !go1.15 go1.21 /* * Copyright 2021 ByteDance Inc. @@ -120,7 +119,6 @@ func (cfg frozenConfig) Valid(data []byte) bool { return json.Valid(data) } - // Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in // order to reduce the first-hit latency at **amd64** Arch. // Opts are the compile options, for example, "option.WithCompileRecursiveDepth" is diff --git a/compat_test.go b/compat_test.go index 1ce0429a3..7dc6c920e 100644 --- a/compat_test.go +++ b/compat_test.go @@ -1,6 +1,3 @@ -//go:build !amd64 -// +build !amd64 - /* * Copyright 2022 ByteDance Inc. * diff --git a/decode_float_test.go b/decode_float_test.go index 009495bde..ba625683d 100644 --- a/decode_float_test.go +++ b/decode_float_test.go @@ -1,4 +1,4 @@ -// +build amd64 +// +build amd64,go1.15,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/decode_test.go b/decode_test.go index b3414c877..cac964d6b 100644 --- a/decode_test.go +++ b/decode_test.go @@ -1,5 +1,4 @@ -//go:build amd64 -// +build amd64 +// +build amd64,go1.15,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/encode_test.go b/encode_test.go index b19b96d5a..b1869b81f 100644 --- a/encode_test.go +++ b/encode_test.go @@ -1,4 +1,4 @@ -// +build amd64 +// +build amd64,go1.15,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/search_test.go b/search_test.go index 68223e45f..c5cfab2a3 100644 --- a/search_test.go +++ b/search_test.go @@ -1,4 +1,4 @@ -// +build amd64 +// +build amd64,go1.15,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/sonic.go b/sonic.go index 829e46161..fe0c7d554 100644 --- a/sonic.go +++ b/sonic.go @@ -1,4 +1,4 @@ -// +build amd64 +// +build amd64,go1.15,!go1.21 /* * Copyright 2021 ByteDance Inc. diff --git a/use_number_test.go b/use_number_test.go index a0197c40d..9673a099a 100644 --- a/use_number_test.go +++ b/use_number_test.go @@ -1,4 +1,4 @@ -// +build amd64 +// +build amd64,go1.15,!go1.21 /* * Copyright 2021 ByteDance Inc. From eb371cda0d1289dc3dc971941b51112d0c1bfbca Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 8 Feb 2023 15:02:06 +0800 Subject: [PATCH 2/3] add compact test --- .github/workflows/push-check-linux-arm64.yml | 2 +- .../push-check-linux-x64-compact.yml | 31 ++++ compat_test.go | 134 ++++++++++++++++++ external_jsonlib_test/unit_test/api_test.go | 131 ----------------- 4 files changed, 166 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/push-check-linux-x64-compact.yml diff --git a/.github/workflows/push-check-linux-arm64.yml b/.github/workflows/push-check-linux-arm64.yml index 9ee5f804c..2947da4b4 100644 --- a/.github/workflows/push-check-linux-arm64.yml +++ b/.github/workflows/push-check-linux-arm64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.15.x, 1.20.x] + go-version: [1.14.x, 1.15.x, 1.20.x] os: [arm] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/push-check-linux-x64-compact.yml b/.github/workflows/push-check-linux-x64-compact.yml new file mode 100644 index 000000000..562bf4e55 --- /dev/null +++ b/.github/workflows/push-check-linux-x64-compact.yml @@ -0,0 +1,31 @@ +name: Push Check Linux-X64 + +on: push + +jobs: + build: + strategy: + matrix: + go-version: [1.14.x] + runs-on: [self-hosted, X64] + steps: + - name: Clear repository + run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE + + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Unit Test + run: | + go test -v -gcflags=-d=checkptr=0 . diff --git a/compat_test.go b/compat_test.go index 7dc6c920e..0c8b0bee1 100644 --- a/compat_test.go +++ b/compat_test.go @@ -17,12 +17,146 @@ package sonic import ( + `bytes` + `encoding/json` `reflect` `testing` `github.com/bytedance/sonic/option` + `github.com/stretchr/testify/require` ) +func TestCompatUnmarshalStd(t *testing.T) { + var sobj = map[string]interface{}{} + var jobj = map[string]interface{}{} + var data = []byte(`{"a":1.00000001E-10}`) + var str = string(data) + serr := ConfigStd.UnmarshalFromString(str, &sobj) + jerr := json.Unmarshal(data, &jobj) + require.Equal(t, jerr, serr) + require.Equal(t, jobj, sobj) + data[2] = '0' + require.Equal(t, jobj, sobj) + + sobj = map[string]interface{}{} + jobj = map[string]interface{}{} + data = []byte(`{"a":1}`) + cfg := Config{ + UseNumber: true, + }.Froze() + serr = cfg.Unmarshal(data, &sobj) + dec := json.NewDecoder(bytes.NewBuffer(data)) + dec.UseNumber() + jerr = dec.Decode(&jobj) + require.Equal(t, jerr, serr) + require.Equal(t, jobj, sobj) + + x := struct{ + A json.Number + B json.Number + }{} + y := struct{ + A json.Number + B json.Number + }{} + data = []byte(`{"A":"1", "C":-1, "B":1}`) + cfg = Config{ + DisallowUnknownFields: true, + }.Froze() + serr = cfg.Unmarshal(data, &x) + dec = json.NewDecoder(bytes.NewBuffer(data)) + dec.UseNumber() + dec.DisallowUnknownFields() + jerr = dec.Decode(&y) + require.Equal(t, jerr, serr) + // require.Equal(t, y, x) +} + +func TestCompatMarshalStd(t *testing.T) { + t.Parallel() + var obj = map[string]interface{}{ + "c": json.RawMessage(" [ \"<&>\" ] "), + "b": json.RawMessage(" [ ] "), + } + sout, serr := ConfigStd.Marshal(obj) + jout, jerr := json.Marshal(obj) + require.Equal(t, jerr, serr) + require.Equal(t, string(jout), string(sout)) + + obj = map[string]interface{}{ + "a": json.RawMessage(" [} "), + } + sout, serr = ConfigStd.Marshal(obj) + jout, jerr = json.Marshal(obj) + require.NotNil(t, jerr) + require.NotNil(t, serr) + require.Equal(t, string(jout), string(sout)) + + obj = map[string]interface{}{ + "a": json.RawMessage("1"), + } + sout, serr = ConfigStd.MarshalIndent(obj, "xxxx", " ") + jout, jerr = json.MarshalIndent(obj, "xxxx", " ") + require.Equal(t, jerr, serr) + require.Equal(t, string(jout), string(sout)) +} + +func TestCompatEncoderStd(t *testing.T) { + var o = map[string]interface{}{ + "a": "<>", + "b": json.RawMessage(" [ ] "), + } + var w1 = bytes.NewBuffer(nil) + var w2 = bytes.NewBuffer(nil) + var enc1 = json.NewEncoder(w1) + var enc2 = ConfigStd.NewEncoder(w2) + + require.Nil(t, enc1.Encode(o)) + require.Nil(t, enc2.Encode(o)) + require.Equal(t, w1.String(), w2.String()) + + enc1.SetEscapeHTML(true) + enc2.SetEscapeHTML(true) + enc1.SetIndent("\n", " ") + enc2.SetIndent("\n", " ") + require.Nil(t, enc1.Encode(o)) + require.Nil(t, enc2.Encode(o)) + require.Equal(t, w1.String(), w2.String()) + + enc1.SetEscapeHTML(false) + enc2.SetEscapeHTML(false) + enc1.SetIndent("", "") + enc2.SetIndent("", "") + require.Nil(t, enc1.Encode(o)) + require.Nil(t, enc2.Encode(o)) + require.Equal(t, w1.String(), w2.String()) +} + +func TestCompatDecoderStd(t *testing.T) { + var o1 = map[string]interface{}{} + var o2 = map[string]interface{}{} + var s = `{"a":"b"} {"1":"2"} a {}` + var w1 = bytes.NewBuffer([]byte(s)) + var w2 = bytes.NewBuffer([]byte(s)) + var enc1 = json.NewDecoder(w1) + var enc2 = ConfigStd.NewDecoder(w2) + + require.Equal(t, enc1.More(), enc2.More()) + require.Nil(t, enc1.Decode(&o1)) + require.Nil(t, enc2.Decode(&o2)) + require.Equal(t, w1.String(), w2.String()) + + require.Equal(t, enc1.More(), enc2.More()) + require.Nil(t, enc1.Decode(&o1)) + require.Nil(t, enc2.Decode(&o2)) + require.Equal(t, w1.String(), w2.String()) + + require.Equal(t, enc1.More(), enc2.More()) + require.NotNil(t, enc1.Decode(&o1)) + require.NotNil(t, enc2.Decode(&o2)) + require.Equal(t, w1.String(), w2.String()) +} + func TestPretouch(t *testing.T) { var v map[string]interface{} if err := Pretouch(reflect.TypeOf(v)); err != nil { diff --git a/external_jsonlib_test/unit_test/api_test.go b/external_jsonlib_test/unit_test/api_test.go index 7e4c8086c..5a571ff3b 100644 --- a/external_jsonlib_test/unit_test/api_test.go +++ b/external_jsonlib_test/unit_test/api_test.go @@ -82,35 +82,6 @@ func TestCompatMarshalDefault(t *testing.T){ require.Equal(t, string(jout), string(sout)) } -func TestCompatMarshalStd(t *testing.T) { - t.Parallel() - var obj = map[string]interface{}{ - "c": json.RawMessage(" [ \"<&>\" ] "), - "b": json.RawMessage(" [ ] "), - } - sout, serr := sonic.ConfigStd.Marshal(obj) - jout, jerr := json.Marshal(obj) - require.Equal(t, jerr, serr) - require.Equal(t, string(jout), string(sout)) - - obj = map[string]interface{}{ - "a": json.RawMessage(" [} "), - } - sout, serr = sonic.ConfigStd.Marshal(obj) - jout, jerr = json.Marshal(obj) - require.NotNil(t, jerr) - require.NotNil(t, serr) - require.Equal(t, string(jout), string(sout)) - - obj = map[string]interface{}{ - "a": json.RawMessage("1"), - } - sout, serr = sonic.ConfigStd.MarshalIndent(obj, "xxxx", " ") - jout, jerr = json.MarshalIndent(obj, "xxxx", " ") - require.Equal(t, jerr, serr) - require.Equal(t, string(jout), string(sout)) -} - func TestCompatUnmarshalDefault(t *testing.T) { var sobj = map[string]interface{}{} var jobj = map[string]interface{}{} @@ -130,52 +101,6 @@ func TestCompatUnmarshalDefault(t *testing.T) { require.Equal(t, y, x) } -func TestCompatUnmarshalStd(t *testing.T) { - var sobj = map[string]interface{}{} - var jobj = map[string]interface{}{} - var data = []byte(`{"a":1.00000001E-10}`) - var str = string(data) - serr := sonic.ConfigStd.UnmarshalFromString(str, &sobj) - jerr := json.Unmarshal(data, &jobj) - require.Equal(t, jerr, serr) - require.Equal(t, jobj, sobj) - data[2] = '0' - require.Equal(t, jobj, sobj) - - sobj = map[string]interface{}{} - jobj = map[string]interface{}{} - data = []byte(`{"a":1}`) - cfg := sonic.Config{ - UseNumber: true, - }.Froze() - serr = cfg.Unmarshal(data, &sobj) - dec := json.NewDecoder(bytes.NewBuffer(data)) - dec.UseNumber() - jerr = dec.Decode(&jobj) - require.Equal(t, jerr, serr) - require.Equal(t, jobj, sobj) - - x := struct{ - A json.Number - B json.Number - }{} - y := struct{ - A json.Number - B json.Number - }{} - data = []byte(`{"A":"1", "C":-1, "B":1}`) - cfg = sonic.Config{ - DisallowUnknownFields: true, - }.Froze() - serr = cfg.Unmarshal(data, &x) - dec = json.NewDecoder(bytes.NewBuffer(data)) - dec.UseNumber() - dec.DisallowUnknownFields() - jerr = dec.Decode(&y) - require.Equal(t, jerr, serr) - // require.Equal(t, y, x) -} - func TestCompatEncoderDefault(t *testing.T) { var o = map[string]interface{}{ "a": "<>", @@ -207,62 +132,6 @@ func TestCompatEncoderDefault(t *testing.T) { require.Equal(t, w1.String(), w2.String()) } -func TestCompatEncoderStd(t *testing.T) { - var o = map[string]interface{}{ - "a": "<>", - "b": json.RawMessage(" [ ] "), - } - var w1 = bytes.NewBuffer(nil) - var w2 = bytes.NewBuffer(nil) - var enc1 = json.NewEncoder(w1) - var enc2 = sonic.ConfigStd.NewEncoder(w2) - - require.Nil(t, enc1.Encode(o)) - require.Nil(t, enc2.Encode(o)) - require.Equal(t, w1.String(), w2.String()) - - enc1.SetEscapeHTML(true) - enc2.SetEscapeHTML(true) - enc1.SetIndent("\n", " ") - enc2.SetIndent("\n", " ") - require.Nil(t, enc1.Encode(o)) - require.Nil(t, enc2.Encode(o)) - require.Equal(t, w1.String(), w2.String()) - - enc1.SetEscapeHTML(false) - enc2.SetEscapeHTML(false) - enc1.SetIndent("", "") - enc2.SetIndent("", "") - require.Nil(t, enc1.Encode(o)) - require.Nil(t, enc2.Encode(o)) - require.Equal(t, w1.String(), w2.String()) -} - -func TestCompatDecoderStd(t *testing.T) { - var o1 = map[string]interface{}{} - var o2 = map[string]interface{}{} - var s = `{"a":"b"} {"1":"2"} a {}` - var w1 = bytes.NewBuffer([]byte(s)) - var w2 = bytes.NewBuffer([]byte(s)) - var enc1 = json.NewDecoder(w1) - var enc2 = sonic.ConfigStd.NewDecoder(w2) - - require.Equal(t, enc1.More(), enc2.More()) - require.Nil(t, enc1.Decode(&o1)) - require.Nil(t, enc2.Decode(&o2)) - require.Equal(t, w1.String(), w2.String()) - - require.Equal(t, enc1.More(), enc2.More()) - require.Nil(t, enc1.Decode(&o1)) - require.Nil(t, enc2.Decode(&o2)) - require.Equal(t, w1.String(), w2.String()) - - require.Equal(t, enc1.More(), enc2.More()) - require.NotNil(t, enc1.Decode(&o1)) - require.NotNil(t, enc2.Decode(&o2)) - require.Equal(t, w1.String(), w2.String()) -} - func TestCompatDecoderDefault(t *testing.T) { var o1 = map[string]interface{}{} var o2 = map[string]interface{}{} From 9ffe67646a3e2fcf50b8ffcf6bb0d8cc8057c405 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Wed, 8 Feb 2023 15:19:45 +0800 Subject: [PATCH 3/3] not support go version <= 1.14 --- .github/workflows/push-check-linux-arm64.yml | 2 +- .../push-check-linux-x64-compact.yml | 31 ------------------- ast/api_amd64_test.go | 3 +- ast/api_compat.go | 2 +- compat.go | 2 +- 5 files changed, 4 insertions(+), 36 deletions(-) delete mode 100644 .github/workflows/push-check-linux-x64-compact.yml diff --git a/.github/workflows/push-check-linux-arm64.yml b/.github/workflows/push-check-linux-arm64.yml index 2947da4b4..9ee5f804c 100644 --- a/.github/workflows/push-check-linux-arm64.yml +++ b/.github/workflows/push-check-linux-arm64.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - go-version: [1.14.x, 1.15.x, 1.20.x] + go-version: [1.15.x, 1.20.x] os: [arm] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/push-check-linux-x64-compact.yml b/.github/workflows/push-check-linux-x64-compact.yml deleted file mode 100644 index 562bf4e55..000000000 --- a/.github/workflows/push-check-linux-x64-compact.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Push Check Linux-X64 - -on: push - -jobs: - build: - strategy: - matrix: - go-version: [1.14.x] - runs-on: [self-hosted, X64] - steps: - - name: Clear repository - run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE - - - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Unit Test - run: | - go test -v -gcflags=-d=checkptr=0 . diff --git a/ast/api_amd64_test.go b/ast/api_amd64_test.go index aacaf75b1..7301f4a3a 100644 --- a/ast/api_amd64_test.go +++ b/ast/api_amd64_test.go @@ -1,5 +1,4 @@ -//go:build amd64 -// +build amd64 +// +build amd64,go1.15,!go1.21 package ast diff --git a/ast/api_compat.go b/ast/api_compat.go index 5eebf6802..642330c51 100644 --- a/ast/api_compat.go +++ b/ast/api_compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.15 go1.21 +// +build !amd64 go1.21 package ast diff --git a/compat.go b/compat.go index e2f67c16d..015aa62bf 100644 --- a/compat.go +++ b/compat.go @@ -1,4 +1,4 @@ -// +build !amd64 !go1.15 go1.21 +// +build !amd64 go1.21 /* * Copyright 2021 ByteDance Inc.