test: enhance data race detection

.github/workflows/unit_test-linux-x64.yml

@@ -5,6 +5,7 @@ on: push
 jobs:
   build:
     strategy:
+      max-parallel: 4
       matrix:
         go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
     runs-on: [self-hosted, X64]
@@ -28,9 +29,15 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Data Race
+        run: |
+          ./scripts/test_race.sh
+
       - name: PCSP Test
         run: python3 ./scripts/test_pcsp.py
 
+
+
       - name: Unit Test
         run: |
           go test -race -covermode=atomic -coverprofile=coverage.txt ./...

internal/decoder/decode_norace.go

@@ -0,0 +1,73 @@
+//go:build !race
+// +build !race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package decoder
+
+import (
+    `unsafe`
+    `encoding/json`
+    `reflect`
+    `runtime`
+
+    `github.com/bytedance/sonic/internal/rt`
+    `github.com/bytedance/sonic/utf8`
+)
+
+// Decode parses the JSON-encoded data from current position and stores the result
+// in the value pointed to by val.
+func (self *Decoder) Decode(val interface{}) error {
+    /* validate json if needed */
+    if (self.f & (1 << _F_validate_string)) != 0  && !utf8.ValidateString(self.s){
+        dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
+        self.s = rt.Mem2Str(dbuf)
+    }
+
+    vv := rt.UnpackEface(val)
+    vp := vv.Value
+
+    /* check for nil type */
+    if vv.Type == nil {
+        return &json.InvalidUnmarshalError{}
+    }
+
+    /* must be a non-nil pointer */
+    if vp == nil || vv.Type.Kind() != reflect.Ptr {
+        return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
+    }
+
+    etp := rt.PtrElem(vv.Type)
+
+    /* check the defined pointer type for issue 379 */
+    if vv.Type.IsNamed() {
+        newp := vp
+        etp  = vv.Type
+        vp   = unsafe.Pointer(&newp)
+    }
+
+    /* create a new stack, and call the decoder */
+    sb := newStack()
+    nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
+    /* return the stack back */
+    self.i = nb
+    freeStack(sb)
+
+    /* avoid GC ahead */
+    runtime.KeepAlive(vv)
+    return err
+}

internal/decoder/decode_race.go

@@ -0,0 +1,80 @@
+//go:build race
+// +build race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package decoder
+
+import (
+    `unsafe`
+    `encoding/json`
+    `reflect`
+    `runtime`
+
+    `github.com/bytedance/sonic/internal/rt`
+    `github.com/bytedance/sonic/utf8`
+)
+
+func helpDetectDataRace(val interface{}) {
+    _, _ = json.Marshal(val)
+}
+
+// Decode parses the JSON-encoded data from current position and stores the result
+// in the value pointed to by val.
+func (self *Decoder) Decode(val interface{}) error {
+    /* validate json if needed */
+    if (self.f & (1 << _F_validate_string)) != 0  && !utf8.ValidateString(self.s){
+        dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
+        self.s = rt.Mem2Str(dbuf)
+    }
+
+    vv := rt.UnpackEface(val)
+    vp := vv.Value
+
+    /* check for nil type */
+    if vv.Type == nil {
+        return &json.InvalidUnmarshalError{}
+    }
+
+    /* must be a non-nil pointer */
+    if vp == nil || vv.Type.Kind() != reflect.Ptr {
+        return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
+    }
+
+    etp := rt.PtrElem(vv.Type)
+
+    /* check the defined pointer type for issue 379 */
+    if vv.Type.IsNamed() {
+        newp := vp
+        etp  = vv.Type
+        vp   = unsafe.Pointer(&newp)
+    }
+
+    /* create a new stack, and call the decoder */
+    sb := newStack()
+    nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
+    /* return the stack back */
+    self.i = nb
+    freeStack(sb)
+
+    /* avoid GC ahead */
+    runtime.KeepAlive(vv)
+
+    /* put last to make the panic from sonic will always be caught at first */
+    helpDetectDataRace(val)
+    return err
+}

internal/decoder/decoder.go

@@ -17,16 +17,12 @@
 package decoder
 
 import (
-    `unsafe`
-    `encoding/json`
     `reflect`
-    `runtime`
 
     `github.com/bytedance/sonic/internal/native`
     `github.com/bytedance/sonic/internal/native/types`
     `github.com/bytedance/sonic/internal/rt`
     `github.com/bytedance/sonic/option`
-    `github.com/bytedance/sonic/utf8`
 )
 
 const (
@@ -106,49 +102,6 @@ func (self *Decoder) CheckTrailings() error {
 }
 
 
-// Decode parses the JSON-encoded data from current position and stores the result
-// in the value pointed to by val.
-func (self *Decoder) Decode(val interface{}) error {
-    /* validate json if needed */
-    if (self.f & (1 << _F_validate_string)) != 0  && !utf8.ValidateString(self.s){
-        dbuf := utf8.CorrectWith(nil, rt.Str2Mem(self.s), "\ufffd")
-        self.s = rt.Mem2Str(dbuf)
-    }
-
-    vv := rt.UnpackEface(val)
-    vp := vv.Value
-
-    /* check for nil type */
-    if vv.Type == nil {
-        return &json.InvalidUnmarshalError{}
-    }
-
-    /* must be a non-nil pointer */
-    if vp == nil || vv.Type.Kind() != reflect.Ptr {
-        return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
-    }
-
-    etp := rt.PtrElem(vv.Type)
-
-    /* check the defined pointer type for issue 379 */
-    if vv.Type.IsNamed() {
-        newp := vp
-        etp  = vv.Type
-        vp   = unsafe.Pointer(&newp)
-    }
-
-    /* create a new stack, and call the decoder */
-    sb := newStack()
-    nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
-    /* return the stack back */
-    self.i = nb
-    freeStack(sb)
-
-    /* avoid GC ahead */
-    runtime.KeepAlive(vv)
-    return err
-}
-
 // UseInt64 indicates the Decoder to unmarshal an integer into an interface{} as an
 // int64 instead of as a float64.
 func (self *Decoder) UseInt64() {

internal/encoder/encode_norace.go

@@ -0,0 +1,44 @@
+//go:build !race
+// +build !race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `runtime`
+
+    `github.com/bytedance/sonic/internal/rt`
+)
+
+
+func encodeInto(buf *[]byte, val interface{}, opts Options) error {
+    stk := newStack()
+    efv := rt.UnpackEface(val)
+    err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
+
+    /* return the stack into pool */
+    if err != nil {
+        resetStack(stk)
+    }
+    freeStack(stk)
+
+    /* avoid GC ahead */
+    runtime.KeepAlive(buf)
+    runtime.KeepAlive(efv)
+    return err
+}

internal/encoder/encode_race.go

@@ -0,0 +1,55 @@
+
+//go:build race
+// +build race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `encoding/json`
+    `runtime`
+
+    `github.com/bytedance/sonic/internal/rt`
+)
+
+
+func helpDetectDataRace(val interface{}) {
+    _, _ = json.Marshal(val)
+}
+
+func encodeInto(buf *[]byte, val interface{}, opts Options) error {
+
+
+    stk := newStack()
+    efv := rt.UnpackEface(val)
+    err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
+
+    /* return the stack into pool */
+    if err != nil {
+        resetStack(stk)
+    }
+    freeStack(stk)
+
+    /* avoid GC ahead */
+    runtime.KeepAlive(buf)
+    runtime.KeepAlive(efv)
+
+    /* put last to make the panic from sonic will always be caught at first */
+    helpDetectDataRace(val)
+    return err
+}

internal/encoder/encoder.go

@@ -20,7 +20,6 @@ import (
     `bytes`
     `encoding/json`
     `reflect`
-    `runtime`
     `unsafe`
 
     `github.com/bytedance/sonic/internal/native`
@@ -233,23 +232,6 @@ func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
     return err
 }
 
-func encodeInto(buf *[]byte, val interface{}, opts Options) error {
-    stk := newStack()
-    efv := rt.UnpackEface(val)
-    err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
-
-    /* return the stack into pool */
-    if err != nil {
-        resetStack(stk)
-    }
-    freeStack(stk)
-
-    /* avoid GC ahead */
-    runtime.KeepAlive(buf)
-    runtime.KeepAlive(efv)
-    return err
-}
-
 func encodeFinish(buf []byte, opts Options) []byte {
     if opts & EscapeHTML != 0 {
         buf = HTMLEscape(nil, buf)

issue_test/issue634_test.go

@@ -1,3 +1,6 @@
+//go:build !race
+// +build !race
+
 /*
  * Copyright 2024 ByteDance Inc.
  *