test: enhance data race detection (#651)

.github/workflows/unit_test-linux-x64.yml

@@ -5,6 +5,7 @@ on: push
 jobs:
   build:
     strategy:
+      max-parallel: 4
       matrix:
         go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
     runs-on: [self-hosted, X64]
@@ -28,9 +29,15 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Data Race
+        run: |
+          ./scripts/test_race.sh
+
       - name: PCSP Test
         run: python3 ./scripts/test_pcsp.py
 
+
+
       - name: Unit Test
         run: |
           go test -race -covermode=atomic -coverprofile=coverage.txt ./...

encoder/encoder_test.go

@@ -23,7 +23,6 @@ import (
     `runtime`
     `runtime/debug`
     `strconv`
-    `sync`
     `testing`
     `time`
 
@@ -47,36 +46,6 @@ func TestMain(m *testing.M) {
     m.Run()
 }
 
-func TestGC(t *testing.T) {
-    if debugSyncGC {
-        return
-    }
-    out, err := Encode(_GenericValue, 0)
-    if err != nil {
-        t.Fatal(err)
-    }
-    n := len(out)
-    wg := &sync.WaitGroup{}
-    N := 10000
-    for i:=0; i<N; i++ {
-        wg.Add(1)
-        go func (wg *sync.WaitGroup, size int)  {
-            defer wg.Done()
-            out, err := Encode(_GenericValue, 0)
-            if err != nil {
-                t.Error(err)
-                return
-            }
-            if len(out) != size {
-                t.Error(len(out), size)
-                return
-            }
-            runtime.GC()
-        }(wg, n)
-    }
-    wg.Wait()
-}
-
 type sample struct {
     M map[string]interface{}
     S []interface{}

internal/encoder/encode_norace.go

@@ -0,0 +1,44 @@
+//go:build !race
+// +build !race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `runtime`
+
+    `github.com/bytedance/sonic/internal/rt`
+)
+
+
+func encodeInto(buf *[]byte, val interface{}, opts Options) error {
+    stk := newStack()
+    efv := rt.UnpackEface(val)
+    err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
+
+    /* return the stack into pool */
+    if err != nil {
+        resetStack(stk)
+    }
+    freeStack(stk)
+
+    /* avoid GC ahead */
+    runtime.KeepAlive(buf)
+    runtime.KeepAlive(efv)
+    return err
+}

internal/encoder/encode_race.go

@@ -0,0 +1,54 @@
+//go:build race
+// +build race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `encoding/json`
+    `runtime`
+
+    `github.com/bytedance/sonic/internal/rt`
+)
+
+
+func helpDetectDataRace(val interface{}) {
+    _, _ = json.Marshal(val)
+}
+
+func encodeInto(buf *[]byte, val interface{}, opts Options) error {
+
+
+    stk := newStack()
+    efv := rt.UnpackEface(val)
+    err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
+
+    /* return the stack into pool */
+    if err != nil {
+        resetStack(stk)
+    }
+    freeStack(stk)
+
+    /* avoid GC ahead */
+    runtime.KeepAlive(buf)
+    runtime.KeepAlive(efv)
+
+    /* put last to make the panic from sonic will always be caught at first */
+    helpDetectDataRace(val)
+    return err
+}

internal/encoder/encoder.go

@@ -20,7 +20,6 @@ import (
     `bytes`
     `encoding/json`
     `reflect`
-    `runtime`
     `unsafe`
 
     `github.com/bytedance/sonic/internal/native`
@@ -233,23 +232,6 @@ func EncodeInto(buf *[]byte, val interface{}, opts Options) error {
     return err
 }
 
-func encodeInto(buf *[]byte, val interface{}, opts Options) error {
-    stk := newStack()
-    efv := rt.UnpackEface(val)
-    err := encodeTypedPointer(buf, efv.Type, &efv.Value, stk, uint64(opts))
-
-    /* return the stack into pool */
-    if err != nil {
-        resetStack(stk)
-    }
-    freeStack(stk)
-
-    /* avoid GC ahead */
-    runtime.KeepAlive(buf)
-    runtime.KeepAlive(efv)
-    return err
-}
-
 func encodeFinish(buf []byte, opts Options) []byte {
     if opts & EscapeHTML != 0 {
         buf = HTMLEscape(nil, buf)

internal/encoder/encoder_norace_test.go

@@ -0,0 +1,55 @@
+//go:build !race
+// +build !race
+
+/*
+ * Copyright 2021 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package encoder
+
+import (
+    `runtime`
+
+    `sync`
+    `testing`
+)
+
+func TestGC(t *testing.T) {
+    if debugSyncGC {
+        return
+    }
+    out, err := Encode(_GenericValue, 0)
+    if err != nil {
+        t.Fatal(err)
+    }
+    n := len(out)
+    wg := &sync.WaitGroup{}
+    N := 10000
+    for i:=0; i<N; i++ {
+        wg.Add(1)
+        go func (wg *sync.WaitGroup, size int)  {
+            defer wg.Done()
+            out, err := Encode(_GenericValue, 0)
+            if err != nil {
+                t.Fatal(err)
+            }
+            if len(out) != size {
+                t.Fatal(len(out), size)
+            }
+            runtime.GC()
+        }(wg, n)
+    }
+    wg.Wait()
+}

internal/encoder/encoder_test.go

@@ -23,7 +23,6 @@ import (
     `runtime`
     `runtime/debug`
     `strconv`
-    `sync`
     `testing`
     `time`
 
@@ -47,34 +46,6 @@ func TestMain(m *testing.M) {
     m.Run()
 }
 
-func TestGC(t *testing.T) {
-    if debugSyncGC {
-        return
-    }
-    out, err := Encode(_GenericValue, 0)
-    if err != nil {
-        t.Fatal(err)
-    }
-    n := len(out)
-    wg := &sync.WaitGroup{}
-    N := 10000
-    for i:=0; i<N; i++ {
-        wg.Add(1)
-        go func (wg *sync.WaitGroup, size int)  {
-            defer wg.Done()
-            out, err := Encode(_GenericValue, 0)
-            if err != nil {
-                t.Fatal(err)
-            }
-            if len(out) != size {
-                t.Fatal(len(out), size)
-            }
-            runtime.GC()
-        }(wg, n)
-    }
-    wg.Wait()
-}
-
 type sample struct {
     M map[string]interface{}
     S []interface{}

issue_test/issue634_test.go

@@ -1,3 +1,6 @@
+//go:build !race
+// +build !race
+
 /*
  * Copyright 2024 ByteDance Inc.
  *

issue_test/race_test_go

@@ -0,0 +1,50 @@
+
+//go:build race
+// +build race
+
+/*
+ * Copyright 2024 ByteDance Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package issue_test
+
+import (
+    `github.com/bytedance/sonic`
+    `testing`
+)
+
+type MyFoo struct {
+    List []*int64 
+}
+
+func TestRaceEncode(t *testing.T) {
+    f := &MyFoo{
+        List: []*int64{new(int64), new(int64)},
+    }
+
+    go func() {
+        f.List = []*int64{new(int64), new(int64)}
+    }()
+
+    go func() {
+        sonic.Marshal(f)
+    }()
+
+    // encoding/json always detect data race when enabling `-race` here
+    // go func() {
+    //     json.Marshal(f)
+    // }()
+}

scripts/test_race.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -xe
+cd ./issue_test
+mv race_test_go race_test.go
+go test -v -run=TestRaceEncode -race -count=100 . > test_race.log || true
+if ! grep -q "WARNING: DATA RACE" ./test_race.log; then
+    echo "TEST FAILED: should data race here"
+    exit 1
+fi
+mv race_test.go race_test_go
+rm -vrf test_race.log